From ebdfa0610edebbbef1f3f6f418b6d29d837d52fd Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Thu, 29 Feb 2024 09:33:03 +1000 Subject: [PATCH 01/51] Add forwarder commands --- src/SeqCli/Cli/Commands/Forward/DumpCommand.cs | 13 +++++++++++++ src/SeqCli/Cli/Commands/Forward/InstallCommand.cs | 13 +++++++++++++ src/SeqCli/Cli/Commands/Forward/RestartCommand.cs | 13 +++++++++++++ src/SeqCli/Cli/Commands/Forward/RunCommand.cs | 13 +++++++++++++ src/SeqCli/Cli/Commands/Forward/StartCommand.cs | 13 +++++++++++++ src/SeqCli/Cli/Commands/Forward/StatusCommand.cs | 13 +++++++++++++ src/SeqCli/Cli/Commands/Forward/StopCommand.cs | 13 +++++++++++++ src/SeqCli/Cli/Commands/Forward/TruncateCommand.cs | 13 +++++++++++++ .../Cli/Commands/Forward/UninstallCommand.cs | 14 ++++++++++++++ 9 files changed, 118 insertions(+) create mode 100644 src/SeqCli/Cli/Commands/Forward/DumpCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forward/InstallCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forward/RestartCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forward/RunCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forward/StartCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forward/StatusCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forward/StopCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forward/TruncateCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs diff --git a/src/SeqCli/Cli/Commands/Forward/DumpCommand.cs b/src/SeqCli/Cli/Commands/Forward/DumpCommand.cs new file mode 100644 index 00000000..02b7c3cd --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forward/DumpCommand.cs @@ -0,0 +1,13 @@ +using SeqCli.Config; +using SeqCli.Connection; + +namespace SeqCli.Cli.Commands.Forward; + +[Command("forward", "dump", "Print the complete log buffer contents as JSON", + Example = "seqcli forward dump")] +class DumpCommand : Command +{ + public DumpCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + { + } +} \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs new file mode 100644 index 00000000..b544be20 --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs @@ -0,0 +1,13 @@ +using SeqCli.Config; +using SeqCli.Connection; + +namespace SeqCli.Cli.Commands.Forward; + +[Command("forward", "install", "Install the Seq Forwarder as a Windows service", + Example = "seqcli forward install")] +class InstallCommand : Command +{ + public InstallCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + { + } +} \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs b/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs new file mode 100644 index 00000000..8bbda6f7 --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs @@ -0,0 +1,13 @@ +using SeqCli.Config; +using SeqCli.Connection; + +namespace SeqCli.Cli.Commands.Forward; + +[Command("forward", "restart", "Restart the Seq Forwarder Windows service", + Example = "seqcli forward restart")] +class RestartCommand : Command +{ + public RestartCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + { + } +} \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/RunCommand.cs b/src/SeqCli/Cli/Commands/Forward/RunCommand.cs new file mode 100644 index 00000000..e95b0936 --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forward/RunCommand.cs @@ -0,0 +1,13 @@ +using SeqCli.Config; +using SeqCli.Connection; + +namespace SeqCli.Cli.Commands.Forward; + +[Command("forward", "run", "Run the Seq Forwarder server interactively", + Example = "seqcli forward run")] +class RunCommand : Command +{ + public RunCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + { + } +} \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/StartCommand.cs b/src/SeqCli/Cli/Commands/Forward/StartCommand.cs new file mode 100644 index 00000000..41946624 --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forward/StartCommand.cs @@ -0,0 +1,13 @@ +using SeqCli.Config; +using SeqCli.Connection; + +namespace SeqCli.Cli.Commands.Forward; + +[Command("forward", "start", "Start the Seq Forwarder Windows service", + Example = "seqcli forward start")] +class StartCommand : Command +{ + public StartCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + { + } +} \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs b/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs new file mode 100644 index 00000000..e4b408dc --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs @@ -0,0 +1,13 @@ +using SeqCli.Config; +using SeqCli.Connection; + +namespace SeqCli.Cli.Commands.Forward; + +[Command("forward", "status", "Show the status of the Seq Forwarder service", + Example = "seqcli forward status")] +class StatusCommand : Command +{ + public StatusCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + { + } +} \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/StopCommand.cs b/src/SeqCli/Cli/Commands/Forward/StopCommand.cs new file mode 100644 index 00000000..b9e6c514 --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forward/StopCommand.cs @@ -0,0 +1,13 @@ +using SeqCli.Config; +using SeqCli.Connection; + +namespace SeqCli.Cli.Commands.Forward; + +[Command("forward", "stop", "Stop the Seq Forwarder service", + Example = "seqcli forward stop")] +class StopCommand : Command +{ + public StopCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + { + } +} \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/TruncateCommand.cs b/src/SeqCli/Cli/Commands/Forward/TruncateCommand.cs new file mode 100644 index 00000000..5499424f --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forward/TruncateCommand.cs @@ -0,0 +1,13 @@ +using SeqCli.Config; +using SeqCli.Connection; + +namespace SeqCli.Cli.Commands.Forward; + +[Command("forward", "truncate", "Clear the log buffer contents", + Example = "seqcli forward truncate")] +class TruncateCommand : Command +{ + public TruncateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + { + } +} \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs new file mode 100644 index 00000000..8416d5d7 --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs @@ -0,0 +1,14 @@ +using SeqCli.Config; +using SeqCli.Connection; + +namespace SeqCli.Cli.Commands.Forward; + +[Command("forward", "uninstall", "Uninstall the Windows service", + Example = "seqcli forward uninstall")] +internal class UninstallCommand : Command +{ + public UninstallCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + { + } +} + From 8d4bcca7149e372cf09fe22caf83fb55bda21166 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Thu, 29 Feb 2024 09:45:43 +1000 Subject: [PATCH 02/51] Conditional compilation --- src/SeqCli/Cli/Commands/Forward/InstallCommand.cs | 6 +++++- src/SeqCli/Cli/Commands/Forward/RestartCommand.cs | 6 +++++- src/SeqCli/Cli/Commands/Forward/StartCommand.cs | 6 +++++- src/SeqCli/Cli/Commands/Forward/StatusCommand.cs | 6 +++++- src/SeqCli/Cli/Commands/Forward/StopCommand.cs | 6 +++++- .../Cli/Commands/Forward/UninstallCommand.cs | 3 +++ src/SeqCli/SeqCli.csproj | 14 ++++++++++++++ 7 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs index b544be20..2004c358 100644 --- a/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs @@ -3,6 +3,8 @@ namespace SeqCli.Cli.Commands.Forward; +#if Windows + [Command("forward", "install", "Install the Seq Forwarder as a Windows service", Example = "seqcli forward install")] class InstallCommand : Command @@ -10,4 +12,6 @@ class InstallCommand : Command public InstallCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) { } -} \ No newline at end of file +} + +#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs b/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs index 8bbda6f7..7533777d 100644 --- a/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs @@ -3,6 +3,8 @@ namespace SeqCli.Cli.Commands.Forward; +#if Windows + [Command("forward", "restart", "Restart the Seq Forwarder Windows service", Example = "seqcli forward restart")] class RestartCommand : Command @@ -10,4 +12,6 @@ class RestartCommand : Command public RestartCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) { } -} \ No newline at end of file +} + +#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/StartCommand.cs b/src/SeqCli/Cli/Commands/Forward/StartCommand.cs index 41946624..3827c210 100644 --- a/src/SeqCli/Cli/Commands/Forward/StartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forward/StartCommand.cs @@ -3,6 +3,8 @@ namespace SeqCli.Cli.Commands.Forward; +#if Windows + [Command("forward", "start", "Start the Seq Forwarder Windows service", Example = "seqcli forward start")] class StartCommand : Command @@ -10,4 +12,6 @@ class StartCommand : Command public StartCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) { } -} \ No newline at end of file +} + +#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs b/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs index e4b408dc..ce83e847 100644 --- a/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs +++ b/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs @@ -3,6 +3,8 @@ namespace SeqCli.Cli.Commands.Forward; +#if Windows + [Command("forward", "status", "Show the status of the Seq Forwarder service", Example = "seqcli forward status")] class StatusCommand : Command @@ -10,4 +12,6 @@ class StatusCommand : Command public StatusCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) { } -} \ No newline at end of file +} + +#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/StopCommand.cs b/src/SeqCli/Cli/Commands/Forward/StopCommand.cs index b9e6c514..8ecc2ae0 100644 --- a/src/SeqCli/Cli/Commands/Forward/StopCommand.cs +++ b/src/SeqCli/Cli/Commands/Forward/StopCommand.cs @@ -3,6 +3,8 @@ namespace SeqCli.Cli.Commands.Forward; +#if Windows + [Command("forward", "stop", "Stop the Seq Forwarder service", Example = "seqcli forward stop")] class StopCommand : Command @@ -10,4 +12,6 @@ class StopCommand : Command public StopCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) { } -} \ No newline at end of file +} + +#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs index 8416d5d7..d5a16ba3 100644 --- a/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs @@ -3,6 +3,8 @@ namespace SeqCli.Cli.Commands.Forward; +#if Windows + [Command("forward", "uninstall", "Uninstall the Windows service", Example = "seqcli forward uninstall")] internal class UninstallCommand : Command @@ -12,3 +14,4 @@ public UninstallCommand(SeqConnectionFactory connectionFactory, SeqCliConfig con } } +#endif \ No newline at end of file diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index 0ee292ec..aa128458 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -12,7 +12,21 @@ seqcli default enable + true + true + true + + + Windows + + + OSX + + + Linux + + From d1ab1cd09b5a825603d324fe83b69e7ca8b6da77 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Thu, 29 Feb 2024 10:20:37 +1000 Subject: [PATCH 03/51] Change casing of compilation symbols --- .../Cli/Commands/Forward/InstallCommand.cs | 2 +- .../Cli/Commands/Forward/RestartCommand.cs | 2 +- src/SeqCli/Cli/Commands/Forward/RunCommand.cs | 10 ++++ .../Cli/Commands/Forward/StartCommand.cs | 2 +- .../Cli/Commands/Forward/StatusCommand.cs | 2 +- .../Cli/Commands/Forward/StopCommand.cs | 2 +- .../Cli/Commands/Forward/UninstallCommand.cs | 2 +- src/SeqCli/Cli/Features/ListenUriFeature.cs | 13 +++++ src/SeqCli/Cli/Features/StoragePathFeature.cs | 58 +++++++++++++++++++ src/SeqCli/SeqCli.csproj | 4 +- 10 files changed, 89 insertions(+), 8 deletions(-) create mode 100644 src/SeqCli/Cli/Features/ListenUriFeature.cs create mode 100644 src/SeqCli/Cli/Features/StoragePathFeature.cs diff --git a/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs index 2004c358..4288dfab 100644 --- a/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs @@ -3,7 +3,7 @@ namespace SeqCli.Cli.Commands.Forward; -#if Windows +#if WINDOWS [Command("forward", "install", "Install the Seq Forwarder as a Windows service", Example = "seqcli forward install")] diff --git a/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs b/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs index 7533777d..0d6cc8e7 100644 --- a/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs @@ -3,7 +3,7 @@ namespace SeqCli.Cli.Commands.Forward; -#if Windows +#if WINDOWS [Command("forward", "restart", "Restart the Seq Forwarder Windows service", Example = "seqcli forward restart")] diff --git a/src/SeqCli/Cli/Commands/Forward/RunCommand.cs b/src/SeqCli/Cli/Commands/Forward/RunCommand.cs index e95b0936..f2b0ed66 100644 --- a/src/SeqCli/Cli/Commands/Forward/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forward/RunCommand.cs @@ -1,3 +1,4 @@ +using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Connection; @@ -7,7 +8,16 @@ namespace SeqCli.Cli.Commands.Forward; Example = "seqcli forward run")] class RunCommand : Command { +#pragma warning disable CS0414 // Field is assigned but its value is never used + bool _noLogo; +#pragma warning restore CS0414 // Field is assigned but its value is never used + readonly StoragePathFeature _storagePath; + readonly ListenUriFeature _listenUri; + public RunCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) { + Options.Add("nologo", _ => _noLogo = true); + _storagePath = Enable(); + _listenUri = Enable(); } } \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/StartCommand.cs b/src/SeqCli/Cli/Commands/Forward/StartCommand.cs index 3827c210..6ddb52dc 100644 --- a/src/SeqCli/Cli/Commands/Forward/StartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forward/StartCommand.cs @@ -3,7 +3,7 @@ namespace SeqCli.Cli.Commands.Forward; -#if Windows +#if WINDOWS [Command("forward", "start", "Start the Seq Forwarder Windows service", Example = "seqcli forward start")] diff --git a/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs b/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs index ce83e847..d9f1f736 100644 --- a/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs +++ b/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs @@ -3,7 +3,7 @@ namespace SeqCli.Cli.Commands.Forward; -#if Windows +#if WINDOWS [Command("forward", "status", "Show the status of the Seq Forwarder service", Example = "seqcli forward status")] diff --git a/src/SeqCli/Cli/Commands/Forward/StopCommand.cs b/src/SeqCli/Cli/Commands/Forward/StopCommand.cs index 8ecc2ae0..06bc93b4 100644 --- a/src/SeqCli/Cli/Commands/Forward/StopCommand.cs +++ b/src/SeqCli/Cli/Commands/Forward/StopCommand.cs @@ -3,7 +3,7 @@ namespace SeqCli.Cli.Commands.Forward; -#if Windows +#if WINDOWS [Command("forward", "stop", "Stop the Seq Forwarder service", Example = "seqcli forward stop")] diff --git a/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs index d5a16ba3..a8fa690f 100644 --- a/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs @@ -3,7 +3,7 @@ namespace SeqCli.Cli.Commands.Forward; -#if Windows +#if WINDOWS [Command("forward", "uninstall", "Uninstall the Windows service", Example = "seqcli forward uninstall")] diff --git a/src/SeqCli/Cli/Features/ListenUriFeature.cs b/src/SeqCli/Cli/Features/ListenUriFeature.cs new file mode 100644 index 00000000..15c5352e --- /dev/null +++ b/src/SeqCli/Cli/Features/ListenUriFeature.cs @@ -0,0 +1,13 @@ +namespace SeqCli.Cli.Features; + +class ListenUriFeature : CommandFeature +{ + public string? ListenUri { get; private set; } + + public override void Enable(OptionSet options) + { + options.Add("l=|listen=", + "Set the listen Uri; http://localhost:15341/ is used by default.", + v => ListenUri = v); + } +} \ No newline at end of file diff --git a/src/SeqCli/Cli/Features/StoragePathFeature.cs b/src/SeqCli/Cli/Features/StoragePathFeature.cs new file mode 100644 index 00000000..ce363a44 --- /dev/null +++ b/src/SeqCli/Cli/Features/StoragePathFeature.cs @@ -0,0 +1,58 @@ +using System; +using System.IO; + +namespace SeqCli.Cli.Features; + +class StoragePathFeature : CommandFeature +{ + string? _storageRoot; + + public string StorageRootPath + { + get + { + if (!string.IsNullOrWhiteSpace(_storageRoot)) + return _storageRoot; + + return TryQueryInstalledStorageRoot() ?? GetDefaultStorageRoot(); + } + } + + public string ConfigFilePath => Path.Combine(StorageRootPath, "SeqForwarder.json"); + + public string BufferPath => Path.Combine(StorageRootPath, "Buffer"); + + public override void Enable(OptionSet options) + { + options.Add("s=|storage=", + "Set the folder where data will be stored; " + + "" + GetDefaultStorageRoot() + " is used by default.", + v => _storageRoot = Path.GetFullPath(v)); + } + + static string GetDefaultStorageRoot() + { + return Path.GetFullPath(Path.Combine( +#if WINDOWS + // Common, here, because the service may run as Local Service, which has no obvious home + // directory. + Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), +#else + // Specific to and writable by the current user. + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), +#endif + "Seq", + "Forwarder")); + } + + static string? TryQueryInstalledStorageRoot() + { +#if WINDOWS + // if (Seq.Forwarder.Util.ServiceConfiguration.GetServiceStoragePath( + // Seq.Forwarder.ServiceProcess.SeqForwarderWindowsService.WindowsServiceName, out var storage)) + // return storage; +#endif + + return null; + } +} \ No newline at end of file diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index aa128458..54e9377f 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -18,13 +18,13 @@ - Windows + WINDOWS OSX - Linux + LINUX From 6494bbf65ddedc4c909aa610155a3a0c33ee5e1f Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Thu, 29 Feb 2024 12:46:43 +1000 Subject: [PATCH 04/51] Add forwarder config --- src/SeqCli/Cli/Commands/Forward/RunCommand.cs | 1 + src/SeqCli/Config/SeqCliApiConfig.cs | 6 ++++ src/SeqCli/Config/SeqCliConfig.cs | 6 +++- src/SeqCli/Config/SeqCliConnectionConfig.cs | 3 ++ src/SeqCli/Config/SeqCliDiagnosticConfig.cs | 29 +++++++++++++++++++ src/SeqCli/Config/SeqCliStorageConfig.cs | 6 ++++ 6 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 src/SeqCli/Config/SeqCliApiConfig.cs create mode 100644 src/SeqCli/Config/SeqCliDiagnosticConfig.cs create mode 100644 src/SeqCli/Config/SeqCliStorageConfig.cs diff --git a/src/SeqCli/Cli/Commands/Forward/RunCommand.cs b/src/SeqCli/Cli/Commands/Forward/RunCommand.cs index f2b0ed66..ba8d4c6d 100644 --- a/src/SeqCli/Cli/Commands/Forward/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forward/RunCommand.cs @@ -1,3 +1,4 @@ +using System; using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Connection; diff --git a/src/SeqCli/Config/SeqCliApiConfig.cs b/src/SeqCli/Config/SeqCliApiConfig.cs new file mode 100644 index 00000000..24e45650 --- /dev/null +++ b/src/SeqCli/Config/SeqCliApiConfig.cs @@ -0,0 +1,6 @@ +namespace SeqCli.Config; + +class SeqCliApiConfig +{ + public string ListenUri { get; set; } = "http://localhost:15341"; +} \ No newline at end of file diff --git a/src/SeqCli/Config/SeqCliConfig.cs b/src/SeqCli/Config/SeqCliConfig.cs index dd3b51ca..718835ce 100644 --- a/src/SeqCli/Config/SeqCliConfig.cs +++ b/src/SeqCli/Config/SeqCliConfig.cs @@ -52,7 +52,11 @@ public static void Write(SeqCliConfig data) } public SeqCliConnectionConfig Connection { get; set; } = new SeqCliConnectionConfig(); - public SeqCliOutputConfig Output { get; set; } = new SeqCliOutputConfig(); + public SeqCliOutputConfig Output { get; set; } = new(); + public SeqCliStorageConfig Storage { get; set; } = new(); + public SeqCliDiagnosticConfig Diagnostics { get; set; } = new(); + public SeqCliApiConfig Api { get; set; } = new(); + public Dictionary Profiles { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); } \ No newline at end of file diff --git a/src/SeqCli/Config/SeqCliConnectionConfig.cs b/src/SeqCli/Config/SeqCliConnectionConfig.cs index 072d733e..b6d64eee 100644 --- a/src/SeqCli/Config/SeqCliConnectionConfig.cs +++ b/src/SeqCli/Config/SeqCliConnectionConfig.cs @@ -23,6 +23,9 @@ class SeqCliConnectionConfig const string ProtectedDataPrefix = "pd."; public string ServerUrl { get; set; } = "http://localhost:5341"; + public ulong EventBodyLimitBytes { get; set; } = 256 * 1024; + public ulong RawPayloadLimitBytes { get; set; } = 10 * 1024 * 1024; + public uint? PooledConnectionLifetimeMilliseconds { get; set; } = null; [JsonProperty("apiKey")] public string? EncodedApiKey { get; set; } diff --git a/src/SeqCli/Config/SeqCliDiagnosticConfig.cs b/src/SeqCli/Config/SeqCliDiagnosticConfig.cs new file mode 100644 index 00000000..a5645c76 --- /dev/null +++ b/src/SeqCli/Config/SeqCliDiagnosticConfig.cs @@ -0,0 +1,29 @@ +using System; +using System.IO; +using Serilog.Events; + +namespace SeqCli.Config; + +class SeqCliDiagnosticConfig +{ + public string InternalLogPath { get; set; } = GetDefaultInternalLogPath(); + public LogEventLevel InternalLoggingLevel { get; set; } = LogEventLevel.Information; + public string? InternalLogServerUri { get; set; } + public string? InternalLogServerApiKey { get; set; } + public bool IngestionLogShowDetail { get; set; } + + public static string GetDefaultInternalLogPath() + { + return Path.Combine( +#if WINDOWS + // Common, here, because the service may run as Local Service, which has no obvious home + // directory. + Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), +#else + // Specific to and writable by the current user. + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), +#endif + "Seq", + "Logs"); + } +} \ No newline at end of file diff --git a/src/SeqCli/Config/SeqCliStorageConfig.cs b/src/SeqCli/Config/SeqCliStorageConfig.cs new file mode 100644 index 00000000..61e1223a --- /dev/null +++ b/src/SeqCli/Config/SeqCliStorageConfig.cs @@ -0,0 +1,6 @@ +namespace SeqCli.Config; + +public class SeqCliStorageConfig +{ + public long BufferSizeBytes { get; set; } = 67_108_864; +} \ No newline at end of file From 368412698d0384fedac31685f6e61c68f855a541 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Thu, 29 Feb 2024 14:13:32 +1000 Subject: [PATCH 05/51] Move forwarder config to forwarder. --- src/SeqCli/Cli/Commands/Forward/RunCommand.cs | 2 - src/SeqCli/Cli/Features/StoragePathFeature.cs | 58 ------------------- src/SeqCli/Config/SeqCliConfig.cs | 4 +- src/SeqCli/Config/SeqCliConnectionConfig.cs | 3 - src/SeqCli/Config/SeqCliDiagnosticConfig.cs | 2 +- src/SeqCli/Config/SeqCliForwarderConfig.cs | 12 ++++ src/SeqCli/Config/SeqCliOutputConfig.cs | 2 +- src/SeqCli/Config/SeqCliStorageConfig.cs | 2 +- src/SeqCli/SeqCli.csproj | 14 +++++ 9 files changed, 30 insertions(+), 69 deletions(-) delete mode 100644 src/SeqCli/Cli/Features/StoragePathFeature.cs create mode 100644 src/SeqCli/Config/SeqCliForwarderConfig.cs diff --git a/src/SeqCli/Cli/Commands/Forward/RunCommand.cs b/src/SeqCli/Cli/Commands/Forward/RunCommand.cs index ba8d4c6d..ca1627ed 100644 --- a/src/SeqCli/Cli/Commands/Forward/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forward/RunCommand.cs @@ -12,13 +12,11 @@ class RunCommand : Command #pragma warning disable CS0414 // Field is assigned but its value is never used bool _noLogo; #pragma warning restore CS0414 // Field is assigned but its value is never used - readonly StoragePathFeature _storagePath; readonly ListenUriFeature _listenUri; public RunCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) { Options.Add("nologo", _ => _noLogo = true); - _storagePath = Enable(); _listenUri = Enable(); } } \ No newline at end of file diff --git a/src/SeqCli/Cli/Features/StoragePathFeature.cs b/src/SeqCli/Cli/Features/StoragePathFeature.cs deleted file mode 100644 index ce363a44..00000000 --- a/src/SeqCli/Cli/Features/StoragePathFeature.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System; -using System.IO; - -namespace SeqCli.Cli.Features; - -class StoragePathFeature : CommandFeature -{ - string? _storageRoot; - - public string StorageRootPath - { - get - { - if (!string.IsNullOrWhiteSpace(_storageRoot)) - return _storageRoot; - - return TryQueryInstalledStorageRoot() ?? GetDefaultStorageRoot(); - } - } - - public string ConfigFilePath => Path.Combine(StorageRootPath, "SeqForwarder.json"); - - public string BufferPath => Path.Combine(StorageRootPath, "Buffer"); - - public override void Enable(OptionSet options) - { - options.Add("s=|storage=", - "Set the folder where data will be stored; " + - "" + GetDefaultStorageRoot() + " is used by default.", - v => _storageRoot = Path.GetFullPath(v)); - } - - static string GetDefaultStorageRoot() - { - return Path.GetFullPath(Path.Combine( -#if WINDOWS - // Common, here, because the service may run as Local Service, which has no obvious home - // directory. - Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), -#else - // Specific to and writable by the current user. - Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), -#endif - "Seq", - "Forwarder")); - } - - static string? TryQueryInstalledStorageRoot() - { -#if WINDOWS - // if (Seq.Forwarder.Util.ServiceConfiguration.GetServiceStoragePath( - // Seq.Forwarder.ServiceProcess.SeqForwarderWindowsService.WindowsServiceName, out var storage)) - // return storage; -#endif - - return null; - } -} \ No newline at end of file diff --git a/src/SeqCli/Config/SeqCliConfig.cs b/src/SeqCli/Config/SeqCliConfig.cs index 718835ce..196aadf4 100644 --- a/src/SeqCli/Config/SeqCliConfig.cs +++ b/src/SeqCli/Config/SeqCliConfig.cs @@ -53,9 +53,7 @@ public static void Write(SeqCliConfig data) public SeqCliConnectionConfig Connection { get; set; } = new SeqCliConnectionConfig(); public SeqCliOutputConfig Output { get; set; } = new(); - public SeqCliStorageConfig Storage { get; set; } = new(); - public SeqCliDiagnosticConfig Diagnostics { get; set; } = new(); - public SeqCliApiConfig Api { get; set; } = new(); + public SeqCliForwarderConfig Forwarder { get; set; } = new(); public Dictionary Profiles { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); diff --git a/src/SeqCli/Config/SeqCliConnectionConfig.cs b/src/SeqCli/Config/SeqCliConnectionConfig.cs index b6d64eee..072d733e 100644 --- a/src/SeqCli/Config/SeqCliConnectionConfig.cs +++ b/src/SeqCli/Config/SeqCliConnectionConfig.cs @@ -23,9 +23,6 @@ class SeqCliConnectionConfig const string ProtectedDataPrefix = "pd."; public string ServerUrl { get; set; } = "http://localhost:5341"; - public ulong EventBodyLimitBytes { get; set; } = 256 * 1024; - public ulong RawPayloadLimitBytes { get; set; } = 10 * 1024 * 1024; - public uint? PooledConnectionLifetimeMilliseconds { get; set; } = null; [JsonProperty("apiKey")] public string? EncodedApiKey { get; set; } diff --git a/src/SeqCli/Config/SeqCliDiagnosticConfig.cs b/src/SeqCli/Config/SeqCliDiagnosticConfig.cs index a5645c76..83e2207a 100644 --- a/src/SeqCli/Config/SeqCliDiagnosticConfig.cs +++ b/src/SeqCli/Config/SeqCliDiagnosticConfig.cs @@ -4,7 +4,7 @@ namespace SeqCli.Config; -class SeqCliDiagnosticConfig +public class SeqCliForwarderDiagnosticConfig { public string InternalLogPath { get; set; } = GetDefaultInternalLogPath(); public LogEventLevel InternalLoggingLevel { get; set; } = LogEventLevel.Information; diff --git a/src/SeqCli/Config/SeqCliForwarderConfig.cs b/src/SeqCli/Config/SeqCliForwarderConfig.cs new file mode 100644 index 00000000..5969c506 --- /dev/null +++ b/src/SeqCli/Config/SeqCliForwarderConfig.cs @@ -0,0 +1,12 @@ +namespace SeqCli.Config; + +class SeqCliForwarderConfig +{ + public uint? PooledConnectionLifetimeMilliseconds { get; set; } = null; + public ulong EventBodyLimitBytes { get; set; } = 256 * 1024; + public ulong PayloadLimitBytes { get; set; } = 10 * 1024 * 1024; + + public SeqCliStorageConfig Storage { get; set; } = new(); + public SeqCliForwarderDiagnosticConfig Diagnostics { get; set; } = new(); + public SeqCliApiConfig Api { get; set; } = new(); +} \ No newline at end of file diff --git a/src/SeqCli/Config/SeqCliOutputConfig.cs b/src/SeqCli/Config/SeqCliOutputConfig.cs index 62b3a6f2..4ba79472 100644 --- a/src/SeqCli/Config/SeqCliOutputConfig.cs +++ b/src/SeqCli/Config/SeqCliOutputConfig.cs @@ -14,7 +14,7 @@ namespace SeqCli.Config; -class SeqCliOutputConfig +public class SeqCliOutputConfig { public bool DisableColor { get; set; } public bool ForceColor { get; set; } diff --git a/src/SeqCli/Config/SeqCliStorageConfig.cs b/src/SeqCli/Config/SeqCliStorageConfig.cs index 61e1223a..ae803813 100644 --- a/src/SeqCli/Config/SeqCliStorageConfig.cs +++ b/src/SeqCli/Config/SeqCliStorageConfig.cs @@ -2,5 +2,5 @@ namespace SeqCli.Config; public class SeqCliStorageConfig { - public long BufferSizeBytes { get; set; } = 67_108_864; + public int BufferSizeBytes { get; set; } = 67_108_864; } \ No newline at end of file diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index 54e9377f..03ea7a9c 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -60,4 +60,18 @@ + + + + + + + + + + + + + + From a0e05c70ea3fe23841b9d4b79271002ce8150a8e Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Thu, 29 Feb 2024 14:16:04 +1000 Subject: [PATCH 06/51] Rename configs --- src/SeqCli/Cli/Commands/ConfigCommand.cs | 2 +- src/SeqCli/Cli/Commands/Node/ListCommand.cs | 2 +- src/SeqCli/Cli/Commands/PrintCommand.cs | 2 +- src/SeqCli/Cli/Commands/Profile/CreateCommand.cs | 2 +- src/SeqCli/Cli/Features/OutputFormatFeature.cs | 2 +- .../{SeqCliConnectionConfig.cs => ConnectionConfig.cs} | 2 +- .../{SeqCliApiConfig.cs => ForwarderApiConfig.cs} | 2 +- .../{SeqCliForwarderConfig.cs => ForwarderConfig.cs} | 8 ++++---- ...eqCliStorageConfig.cs => ForwarderStorageConfig.cs} | 2 +- .../Config/{SeqCliOutputConfig.cs => OutputConfig.cs} | 2 +- src/SeqCli/Config/SeqCliConfig.cs | 10 +++++----- src/SeqCli/Config/SeqCliDiagnosticConfig.cs | 2 +- 12 files changed, 19 insertions(+), 19 deletions(-) rename src/SeqCli/Config/{SeqCliConnectionConfig.cs => ConnectionConfig.cs} (98%) rename src/SeqCli/Config/{SeqCliApiConfig.cs => ForwarderApiConfig.cs} (79%) rename src/SeqCli/Config/{SeqCliForwarderConfig.cs => ForwarderConfig.cs} (52%) rename src/SeqCli/Config/{SeqCliStorageConfig.cs => ForwarderStorageConfig.cs} (70%) rename src/SeqCli/Config/{SeqCliOutputConfig.cs => OutputConfig.cs} (95%) diff --git a/src/SeqCli/Cli/Commands/ConfigCommand.cs b/src/SeqCli/Cli/Commands/ConfigCommand.cs index 5ec9beec..f991dc44 100644 --- a/src/SeqCli/Cli/Commands/ConfigCommand.cs +++ b/src/SeqCli/Cli/Commands/ConfigCommand.cs @@ -107,7 +107,7 @@ static void Set(SeqCliConfig config, string key, string? value) if (first == null) throw new ArgumentException("The key could not be found; run the command without any arguments to view all keys."); - if (first.PropertyType == typeof(Dictionary)) + if (first.PropertyType == typeof(Dictionary)) throw new NotSupportedException("Use `seqcli profile create` to configure connection profiles."); var second = first.PropertyType.GetTypeInfo().DeclaredProperties diff --git a/src/SeqCli/Cli/Commands/Node/ListCommand.cs b/src/SeqCli/Cli/Commands/Node/ListCommand.cs index bf030933..9b24b03c 100644 --- a/src/SeqCli/Cli/Commands/Node/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Node/ListCommand.cs @@ -32,7 +32,7 @@ class ListCommand : Command string? _name, _id; - public ListCommand(SeqConnectionFactory connectionFactory, SeqCliOutputConfig outputConfig) + public ListCommand(SeqConnectionFactory connectionFactory, OutputConfig outputConfig) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); diff --git a/src/SeqCli/Cli/Commands/PrintCommand.cs b/src/SeqCli/Cli/Commands/PrintCommand.cs index a5d24dda..2a96e68a 100644 --- a/src/SeqCli/Cli/Commands/PrintCommand.cs +++ b/src/SeqCli/Cli/Commands/PrintCommand.cs @@ -38,7 +38,7 @@ class PrintCommand : Command string? _filter, _template = OutputFormatFeature.DefaultOutputTemplate; bool _noColor, _forceColor; - public PrintCommand(SeqCliOutputConfig outputConfig) + public PrintCommand(OutputConfig outputConfig) { if (outputConfig == null) throw new ArgumentNullException(nameof(outputConfig)); _noColor = outputConfig.DisableColor; diff --git a/src/SeqCli/Cli/Commands/Profile/CreateCommand.cs b/src/SeqCli/Cli/Commands/Profile/CreateCommand.cs index 83d32e21..d6769739 100644 --- a/src/SeqCli/Cli/Commands/Profile/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Profile/CreateCommand.cs @@ -49,7 +49,7 @@ int RunSync() try { var config = SeqCliConfig.Read(); - config.Profiles[_name] = new SeqCliConnectionConfig { ServerUrl = _url, ApiKey = _apiKey }; + config.Profiles[_name] = new ConnectionConfig { ServerUrl = _url, ApiKey = _apiKey }; SeqCliConfig.Write(config); return 0; } diff --git a/src/SeqCli/Cli/Features/OutputFormatFeature.cs b/src/SeqCli/Cli/Features/OutputFormatFeature.cs index f64ef386..ac6a237f 100644 --- a/src/SeqCli/Cli/Features/OutputFormatFeature.cs +++ b/src/SeqCli/Cli/Features/OutputFormatFeature.cs @@ -40,7 +40,7 @@ class OutputFormatFeature : CommandFeature bool _json, _noColor, _forceColor; - public OutputFormatFeature(SeqCliOutputConfig outputConfig) + public OutputFormatFeature(OutputConfig outputConfig) { _noColor = outputConfig.DisableColor; _forceColor = outputConfig.ForceColor; diff --git a/src/SeqCli/Config/SeqCliConnectionConfig.cs b/src/SeqCli/Config/ConnectionConfig.cs similarity index 98% rename from src/SeqCli/Config/SeqCliConnectionConfig.cs rename to src/SeqCli/Config/ConnectionConfig.cs index 072d733e..57b6fd50 100644 --- a/src/SeqCli/Config/SeqCliConnectionConfig.cs +++ b/src/SeqCli/Config/ConnectionConfig.cs @@ -18,7 +18,7 @@ namespace SeqCli.Config; -class SeqCliConnectionConfig +class ConnectionConfig { const string ProtectedDataPrefix = "pd."; diff --git a/src/SeqCli/Config/SeqCliApiConfig.cs b/src/SeqCli/Config/ForwarderApiConfig.cs similarity index 79% rename from src/SeqCli/Config/SeqCliApiConfig.cs rename to src/SeqCli/Config/ForwarderApiConfig.cs index 24e45650..321e8017 100644 --- a/src/SeqCli/Config/SeqCliApiConfig.cs +++ b/src/SeqCli/Config/ForwarderApiConfig.cs @@ -1,6 +1,6 @@ namespace SeqCli.Config; -class SeqCliApiConfig +class ForwarderApiConfig { public string ListenUri { get; set; } = "http://localhost:15341"; } \ No newline at end of file diff --git a/src/SeqCli/Config/SeqCliForwarderConfig.cs b/src/SeqCli/Config/ForwarderConfig.cs similarity index 52% rename from src/SeqCli/Config/SeqCliForwarderConfig.cs rename to src/SeqCli/Config/ForwarderConfig.cs index 5969c506..621072cc 100644 --- a/src/SeqCli/Config/SeqCliForwarderConfig.cs +++ b/src/SeqCli/Config/ForwarderConfig.cs @@ -1,12 +1,12 @@ namespace SeqCli.Config; -class SeqCliForwarderConfig +class ForwarderConfig { public uint? PooledConnectionLifetimeMilliseconds { get; set; } = null; public ulong EventBodyLimitBytes { get; set; } = 256 * 1024; public ulong PayloadLimitBytes { get; set; } = 10 * 1024 * 1024; - public SeqCliStorageConfig Storage { get; set; } = new(); - public SeqCliForwarderDiagnosticConfig Diagnostics { get; set; } = new(); - public SeqCliApiConfig Api { get; set; } = new(); + public ForwarderStorageConfig Storage { get; set; } = new(); + public ForwarderDiagnosticConfig Diagnostics { get; set; } = new(); + public ForwarderApiConfig Api { get; set; } = new(); } \ No newline at end of file diff --git a/src/SeqCli/Config/SeqCliStorageConfig.cs b/src/SeqCli/Config/ForwarderStorageConfig.cs similarity index 70% rename from src/SeqCli/Config/SeqCliStorageConfig.cs rename to src/SeqCli/Config/ForwarderStorageConfig.cs index ae803813..f2143eaa 100644 --- a/src/SeqCli/Config/SeqCliStorageConfig.cs +++ b/src/SeqCli/Config/ForwarderStorageConfig.cs @@ -1,6 +1,6 @@ namespace SeqCli.Config; -public class SeqCliStorageConfig +public class ForwarderStorageConfig { public int BufferSizeBytes { get; set; } = 67_108_864; } \ No newline at end of file diff --git a/src/SeqCli/Config/SeqCliOutputConfig.cs b/src/SeqCli/Config/OutputConfig.cs similarity index 95% rename from src/SeqCli/Config/SeqCliOutputConfig.cs rename to src/SeqCli/Config/OutputConfig.cs index 4ba79472..8727c2b3 100644 --- a/src/SeqCli/Config/SeqCliOutputConfig.cs +++ b/src/SeqCli/Config/OutputConfig.cs @@ -14,7 +14,7 @@ namespace SeqCli.Config; -public class SeqCliOutputConfig +public class OutputConfig { public bool DisableColor { get; set; } public bool ForceColor { get; set; } diff --git a/src/SeqCli/Config/SeqCliConfig.cs b/src/SeqCli/Config/SeqCliConfig.cs index 196aadf4..307f14b8 100644 --- a/src/SeqCli/Config/SeqCliConfig.cs +++ b/src/SeqCli/Config/SeqCliConfig.cs @@ -51,10 +51,10 @@ public static void Write(SeqCliConfig data) File.WriteAllText(DefaultConfigFilename, content); } - public SeqCliConnectionConfig Connection { get; set; } = new SeqCliConnectionConfig(); - public SeqCliOutputConfig Output { get; set; } = new(); - public SeqCliForwarderConfig Forwarder { get; set; } = new(); + public ConnectionConfig Connection { get; set; } = new ConnectionConfig(); + public OutputConfig Output { get; set; } = new(); + public ForwarderConfig Forwarder { get; set; } = new(); - public Dictionary Profiles { get; } = - new Dictionary(StringComparer.OrdinalIgnoreCase); + public Dictionary Profiles { get; } = + new Dictionary(StringComparer.OrdinalIgnoreCase); } \ No newline at end of file diff --git a/src/SeqCli/Config/SeqCliDiagnosticConfig.cs b/src/SeqCli/Config/SeqCliDiagnosticConfig.cs index 83e2207a..8366a6e4 100644 --- a/src/SeqCli/Config/SeqCliDiagnosticConfig.cs +++ b/src/SeqCli/Config/SeqCliDiagnosticConfig.cs @@ -4,7 +4,7 @@ namespace SeqCli.Config; -public class SeqCliForwarderDiagnosticConfig +public class ForwarderDiagnosticConfig { public string InternalLogPath { get; set; } = GetDefaultInternalLogPath(); public LogEventLevel InternalLoggingLevel { get; set; } = LogEventLevel.Information; From 5d7d9cb3a07482bea4e06e3029b9499c1b6fcf2e Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Thu, 29 Feb 2024 14:18:04 +1000 Subject: [PATCH 07/51] Moved forwarder configs to directory --- src/SeqCli/Config/{ => Forwarder}/ForwarderApiConfig.cs | 0 src/SeqCli/Config/{ => Forwarder}/ForwarderConfig.cs | 0 .../ForwarderDiagnosticConfig.cs} | 0 src/SeqCli/Config/{ => Forwarder}/ForwarderStorageConfig.cs | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename src/SeqCli/Config/{ => Forwarder}/ForwarderApiConfig.cs (100%) rename src/SeqCli/Config/{ => Forwarder}/ForwarderConfig.cs (100%) rename src/SeqCli/Config/{SeqCliDiagnosticConfig.cs => Forwarder/ForwarderDiagnosticConfig.cs} (100%) rename src/SeqCli/Config/{ => Forwarder}/ForwarderStorageConfig.cs (100%) diff --git a/src/SeqCli/Config/ForwarderApiConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs similarity index 100% rename from src/SeqCli/Config/ForwarderApiConfig.cs rename to src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs diff --git a/src/SeqCli/Config/ForwarderConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderConfig.cs similarity index 100% rename from src/SeqCli/Config/ForwarderConfig.cs rename to src/SeqCli/Config/Forwarder/ForwarderConfig.cs diff --git a/src/SeqCli/Config/SeqCliDiagnosticConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs similarity index 100% rename from src/SeqCli/Config/SeqCliDiagnosticConfig.cs rename to src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs diff --git a/src/SeqCli/Config/ForwarderStorageConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs similarity index 100% rename from src/SeqCli/Config/ForwarderStorageConfig.cs rename to src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs From 561a5b1c503e277e7d3ec605fdcff7219066107a Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Thu, 29 Feb 2024 14:19:41 +1000 Subject: [PATCH 08/51] Cleanup project file --- src/SeqCli/SeqCli.csproj | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index 03ea7a9c..54e9377f 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -60,18 +60,4 @@ - - - - - - - - - - - - - - From f828da64531eb425c700ad73f49675b07d6448ae Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 29 Feb 2024 14:31:17 +1000 Subject: [PATCH 09/51] Import existing Seq Forwarder codebase --- seqcli.sln | 1 - .../Cli/Commands/Forward/DumpCommand.cs | 13 - .../Cli/Commands/Forward/InstallCommand.cs | 17 -- .../Cli/Commands/Forward/RestartCommand.cs | 17 -- src/SeqCli/Cli/Commands/Forward/RunCommand.cs | 23 -- .../Cli/Commands/Forward/StartCommand.cs | 17 -- .../Cli/Commands/Forward/StatusCommand.cs | 17 -- .../Cli/Commands/Forward/StopCommand.cs | 17 -- .../Cli/Commands/Forward/TruncateCommand.cs | 13 - .../Cli/Commands/Forward/UninstallCommand.cs | 17 -- .../Cli/Commands/Forwarder/InstallCommand.cs | 262 ++++++++++++++++ .../Cli/Commands/Forwarder/RestartCommand.cs | 83 ++++++ .../Cli/Commands/Forwarder/RunCommand.cs | 258 ++++++++++++++++ .../Cli/Commands/Forwarder/StartCommand.cs | 67 +++++ .../Cli/Commands/Forwarder/StatusCommand.cs | 51 ++++ .../Cli/Commands/Forwarder/StopCommand.cs | 68 +++++ .../Cli/Commands/Forwarder/TruncateCommand.cs | 51 ++++ .../Commands/Forwarder/UninstallCommand.cs | 50 ++++ src/SeqCli/Cli/Features/ListenUriFeature.cs | 2 +- .../Cli/Features/ServiceCredentialsFeature.cs | 42 +++ src/SeqCli/Cli/Features/StoragePathFeature.cs | 6 +- .../Forwarder/Config/SeqForwarderApiConfig.cs | 21 ++ .../Forwarder/Config/SeqForwarderConfig.cs | 95 ++++++ .../Config/SeqForwarderDiagnosticConfig.cs | 44 +++ .../Config/SeqForwarderOutputConfig.cs | 56 ++++ .../Config/SeqForwarderStorageConfig.cs | 21 ++ .../DpapiMachineScopeDataProtection.cs | 46 +++ .../Cryptography/IStringDataProtector.cs | 8 + .../Cryptography/StringDataProtector.cs | 14 + .../Cryptography/UnprotectedStringData.cs | 22 ++ .../Forwarder/Diagnostics/InMemorySink.cs | 49 +++ .../Forwarder/Diagnostics/IngestionLog.cs | 65 ++++ .../Forwarder/Multiplexing/ActiveLogBuffer.cs | 38 +++ .../Multiplexing/ActiveLogBufferMap.cs | 235 +++++++++++++++ .../Multiplexing/HttpLogShipperFactory.cs | 41 +++ .../Multiplexing/ILogShipperFactory.cs | 24 ++ .../Multiplexing/InertLogShipperFactory.cs | 27 ++ .../Multiplexing/ServerResponseProxy.cs | 52 ++++ .../Forwarder/Properties/AssemblyInfo.cs | 3 + src/SeqCli/Forwarder/Schema/EventSchema.cs | 187 ++++++++++++ src/SeqCli/Forwarder/SeqForwarderModule.cs | 86 ++++++ .../SeqForwarderWindowsService.cs | 52 ++++ .../ExponentialBackoffConnectionSchedule.cs | 74 +++++ .../Forwarder/Shipper/HttpLogShipper.cs | 251 ++++++++++++++++ .../Forwarder/Shipper/InertLogShipper.cs | 31 ++ src/SeqCli/Forwarder/Shipper/LogShipper.cs | 25 ++ src/SeqCli/Forwarder/Shipper/SeqApi.cs | 21 ++ src/SeqCli/Forwarder/Storage/LogBuffer.cs | 280 ++++++++++++++++++ .../Forwarder/Storage/LogBufferEntry.cs | 24 ++ .../Forwarder/Util/AccountRightsHelper.cs | 193 ++++++++++++ src/SeqCli/Forwarder/Util/CaptiveProcess.cs | 82 +++++ .../Forwarder/Util/EnumerableExtensions.cs | 20 ++ .../Forwarder/Util/ExecutionEnvironment.cs | 20 ++ .../Forwarder/Util/ServiceConfiguration.cs | 111 +++++++ .../Forwarder/Util/UnclosableStreamWrapper.cs | 60 ++++ src/SeqCli/Forwarder/Util/WindowsProcess.cs | 51 ++++ .../Forwarder/Web/Api/ApiRootController.cs | 57 ++++ .../Forwarder/Web/Api/IngestionController.cs | 246 +++++++++++++++ .../Forwarder/Web/Host/ServerService.cs | 67 +++++ src/SeqCli/Forwarder/Web/Host/Startup.cs | 40 +++ .../Web/RequestProcessingException.cs | 30 ++ src/SeqCli/SeqCli.csproj | 23 +- .../Multiplexing/ActiveLogBufferMapTests.cs | 83 ++++++ .../Forwarder/Schema/EventSchemaTests.cs | 73 +++++ .../Shipper/ServerResponseProxyTests.cs | 49 +++ .../Forwarder/Storage/LogBufferTests.cs | 151 ++++++++++ test/SeqCli.Tests/SeqCli.Tests.csproj | 1 + test/SeqCli.Tests/Support/Some.cs | 17 ++ test/SeqCli.Tests/Support/TempFolder.cs | 51 ++++ 69 files changed, 4237 insertions(+), 172 deletions(-) delete mode 100644 src/SeqCli/Cli/Commands/Forward/DumpCommand.cs delete mode 100644 src/SeqCli/Cli/Commands/Forward/InstallCommand.cs delete mode 100644 src/SeqCli/Cli/Commands/Forward/RestartCommand.cs delete mode 100644 src/SeqCli/Cli/Commands/Forward/RunCommand.cs delete mode 100644 src/SeqCli/Cli/Commands/Forward/StartCommand.cs delete mode 100644 src/SeqCli/Cli/Commands/Forward/StatusCommand.cs delete mode 100644 src/SeqCli/Cli/Commands/Forward/StopCommand.cs delete mode 100644 src/SeqCli/Cli/Commands/Forward/TruncateCommand.cs delete mode 100644 src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs create mode 100644 src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs create mode 100644 src/SeqCli/Forwarder/Config/SeqForwarderApiConfig.cs create mode 100644 src/SeqCli/Forwarder/Config/SeqForwarderConfig.cs create mode 100644 src/SeqCli/Forwarder/Config/SeqForwarderDiagnosticConfig.cs create mode 100644 src/SeqCli/Forwarder/Config/SeqForwarderOutputConfig.cs create mode 100644 src/SeqCli/Forwarder/Config/SeqForwarderStorageConfig.cs create mode 100644 src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs create mode 100644 src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs create mode 100644 src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs create mode 100644 src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs create mode 100644 src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs create mode 100644 src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs create mode 100644 src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs create mode 100644 src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs create mode 100644 src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs create mode 100644 src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs create mode 100644 src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs create mode 100644 src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs create mode 100644 src/SeqCli/Forwarder/Properties/AssemblyInfo.cs create mode 100644 src/SeqCli/Forwarder/Schema/EventSchema.cs create mode 100644 src/SeqCli/Forwarder/SeqForwarderModule.cs create mode 100644 src/SeqCli/Forwarder/ServiceProcess/SeqForwarderWindowsService.cs create mode 100644 src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs create mode 100644 src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs create mode 100644 src/SeqCli/Forwarder/Shipper/InertLogShipper.cs create mode 100644 src/SeqCli/Forwarder/Shipper/LogShipper.cs create mode 100644 src/SeqCli/Forwarder/Shipper/SeqApi.cs create mode 100644 src/SeqCli/Forwarder/Storage/LogBuffer.cs create mode 100644 src/SeqCli/Forwarder/Storage/LogBufferEntry.cs create mode 100644 src/SeqCli/Forwarder/Util/AccountRightsHelper.cs create mode 100644 src/SeqCli/Forwarder/Util/CaptiveProcess.cs create mode 100644 src/SeqCli/Forwarder/Util/EnumerableExtensions.cs create mode 100644 src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs create mode 100644 src/SeqCli/Forwarder/Util/ServiceConfiguration.cs create mode 100644 src/SeqCli/Forwarder/Util/UnclosableStreamWrapper.cs create mode 100644 src/SeqCli/Forwarder/Util/WindowsProcess.cs create mode 100644 src/SeqCli/Forwarder/Web/Api/ApiRootController.cs create mode 100644 src/SeqCli/Forwarder/Web/Api/IngestionController.cs create mode 100644 src/SeqCli/Forwarder/Web/Host/ServerService.cs create mode 100644 src/SeqCli/Forwarder/Web/Host/Startup.cs create mode 100644 src/SeqCli/Forwarder/Web/RequestProcessingException.cs create mode 100644 test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs create mode 100644 test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs create mode 100644 test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs create mode 100644 test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs create mode 100644 test/SeqCli.Tests/Support/TempFolder.cs diff --git a/seqcli.sln b/seqcli.sln index 6c6104c5..bad9ba56 100644 --- a/seqcli.sln +++ b/seqcli.sln @@ -24,7 +24,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{3587B633-0 EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "asset", "asset", "{438A0DA5-F3CF-4FCE-B43A-B6DA2981D4AE}" ProjectSection(SolutionItems) = preProject - asset\SeqCliLicense.rtf = asset\SeqCliLicense.rtf asset\SeqCli.ico = asset\SeqCli.ico asset\SeqCli.png = asset\SeqCli.png EndProjectSection diff --git a/src/SeqCli/Cli/Commands/Forward/DumpCommand.cs b/src/SeqCli/Cli/Commands/Forward/DumpCommand.cs deleted file mode 100644 index 02b7c3cd..00000000 --- a/src/SeqCli/Cli/Commands/Forward/DumpCommand.cs +++ /dev/null @@ -1,13 +0,0 @@ -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -[Command("forward", "dump", "Print the complete log buffer contents as JSON", - Example = "seqcli forward dump")] -class DumpCommand : Command -{ - public DumpCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - } -} \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs deleted file mode 100644 index 4288dfab..00000000 --- a/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs +++ /dev/null @@ -1,17 +0,0 @@ -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -#if WINDOWS - -[Command("forward", "install", "Install the Seq Forwarder as a Windows service", - Example = "seqcli forward install")] -class InstallCommand : Command -{ - public InstallCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - } -} - -#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs b/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs deleted file mode 100644 index 0d6cc8e7..00000000 --- a/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs +++ /dev/null @@ -1,17 +0,0 @@ -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -#if WINDOWS - -[Command("forward", "restart", "Restart the Seq Forwarder Windows service", - Example = "seqcli forward restart")] -class RestartCommand : Command -{ - public RestartCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - } -} - -#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/RunCommand.cs b/src/SeqCli/Cli/Commands/Forward/RunCommand.cs deleted file mode 100644 index f2b0ed66..00000000 --- a/src/SeqCli/Cli/Commands/Forward/RunCommand.cs +++ /dev/null @@ -1,23 +0,0 @@ -using SeqCli.Cli.Features; -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -[Command("forward", "run", "Run the Seq Forwarder server interactively", - Example = "seqcli forward run")] -class RunCommand : Command -{ -#pragma warning disable CS0414 // Field is assigned but its value is never used - bool _noLogo; -#pragma warning restore CS0414 // Field is assigned but its value is never used - readonly StoragePathFeature _storagePath; - readonly ListenUriFeature _listenUri; - - public RunCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - Options.Add("nologo", _ => _noLogo = true); - _storagePath = Enable(); - _listenUri = Enable(); - } -} \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/StartCommand.cs b/src/SeqCli/Cli/Commands/Forward/StartCommand.cs deleted file mode 100644 index 6ddb52dc..00000000 --- a/src/SeqCli/Cli/Commands/Forward/StartCommand.cs +++ /dev/null @@ -1,17 +0,0 @@ -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -#if WINDOWS - -[Command("forward", "start", "Start the Seq Forwarder Windows service", - Example = "seqcli forward start")] -class StartCommand : Command -{ - public StartCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - } -} - -#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs b/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs deleted file mode 100644 index d9f1f736..00000000 --- a/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs +++ /dev/null @@ -1,17 +0,0 @@ -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -#if WINDOWS - -[Command("forward", "status", "Show the status of the Seq Forwarder service", - Example = "seqcli forward status")] -class StatusCommand : Command -{ - public StatusCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - } -} - -#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/StopCommand.cs b/src/SeqCli/Cli/Commands/Forward/StopCommand.cs deleted file mode 100644 index 06bc93b4..00000000 --- a/src/SeqCli/Cli/Commands/Forward/StopCommand.cs +++ /dev/null @@ -1,17 +0,0 @@ -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -#if WINDOWS - -[Command("forward", "stop", "Stop the Seq Forwarder service", - Example = "seqcli forward stop")] -class StopCommand : Command -{ - public StopCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - } -} - -#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/TruncateCommand.cs b/src/SeqCli/Cli/Commands/Forward/TruncateCommand.cs deleted file mode 100644 index 5499424f..00000000 --- a/src/SeqCli/Cli/Commands/Forward/TruncateCommand.cs +++ /dev/null @@ -1,13 +0,0 @@ -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -[Command("forward", "truncate", "Clear the log buffer contents", - Example = "seqcli forward truncate")] -class TruncateCommand : Command -{ - public TruncateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - } -} \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs deleted file mode 100644 index a8fa690f..00000000 --- a/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs +++ /dev/null @@ -1,17 +0,0 @@ -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -#if WINDOWS - -[Command("forward", "uninstall", "Uninstall the Windows service", - Example = "seqcli forward uninstall")] -internal class UninstallCommand : Command -{ - public UninstallCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - } -} - -#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs new file mode 100644 index 00000000..7dbc9873 --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs @@ -0,0 +1,262 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System; +using System.IO; +using System.Runtime.InteropServices; +using System.Security.AccessControl; +using System.ServiceProcess; +using Seq.Forwarder.Cli.Features; +using Seq.Forwarder.Config; +using Seq.Forwarder.ServiceProcess; +using Seq.Forwarder.Util; + +// ReSharper disable once ClassNeverInstantiated.Global + +namespace Seq.Forwarder.Cli.Commands +{ + [Command("forwarder", "install", "Install the Seq Forwarder as a Windows service")] + class InstallCommand : Command + { + readonly StoragePathFeature _storagePath; + readonly ServiceCredentialsFeature _serviceCredentials; + readonly ListenUriFeature _listenUri; + + bool _setup; + + public InstallCommand() + { + _storagePath = Enable(); + _listenUri = Enable(); + _serviceCredentials = Enable(); + + Options.Add( + "setup", + "Install and start the service only if it does not exist; otherwise reconfigure the binary location", + v => _setup = true); + } + + string ServiceUsername => _serviceCredentials.IsUsernameSpecified ? _serviceCredentials.Username : "NT AUTHORITY\\LocalService"; + + protected override int Run(TextWriter cout) + { + try + { + if (!_setup) + { + Install(cout); + return 0; + } + + var exit = Setup(cout); + if (exit == 0) + { + Console.ForegroundColor = ConsoleColor.Green; + Console.WriteLine("Setup completed successfully."); + Console.ResetColor(); + } + return exit; + } + catch (DirectoryNotFoundException dex) + { + cout.WriteLine("Could not install the service, directory not found: " + dex.Message); + return -1; + } + catch (Exception ex) + { + cout.WriteLine("Could not install the service: " + ex.Message); + return -1; + } + } + + int Setup(TextWriter cout) + { + ServiceController controller; + try + { + cout.WriteLine("Checking the status of the Seq Forwarder service..."); + + controller = new ServiceController(SeqForwarderWindowsService.WindowsServiceName); + cout.WriteLine("Status is {0}", controller.Status); + } + catch (InvalidOperationException) + { + Install(cout); + var controller2 = new ServiceController(SeqForwarderWindowsService.WindowsServiceName); + return Start(controller2, cout); + } + + cout.WriteLine("Service is installed; checking path and dependency configuration..."); + Reconfigure(controller, cout); + + if (controller.Status != ServiceControllerStatus.Running) + return Start(controller, cout); + + return 0; + } + + static void Reconfigure(ServiceController controller, TextWriter cout) + { + var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); + if (0 != CaptiveProcess.Run(sc, "config \"" + controller.ServiceName + "\" depend= Winmgmt/Tcpip/CryptSvc", cout.WriteLine, cout.WriteLine)) + cout.WriteLine("Could not reconfigure service dependencies; ignoring."); + + if (!ServiceConfiguration.GetServiceBinaryPath(controller, cout, out var path)) + return; + + var current = "\"" + Path.Combine(AppDomain.CurrentDomain.BaseDirectory!, Program.BinaryName) + "\""; + if (path.StartsWith(current)) + return; + + var seqRun = path.IndexOf(Program.BinaryName + "\" run", StringComparison.OrdinalIgnoreCase); + if (seqRun == -1) + { + cout.WriteLine("Current binary path is an unrecognized format."); + return; + } + + cout.WriteLine("Existing service binary path is: {0}", path); + + var trimmed = path.Substring((seqRun + Program.BinaryName + " ").Length); + var newPath = current + trimmed; + cout.WriteLine("Updating service binary path configuration to: {0}", newPath); + + var escaped = newPath.Replace("\"", "\\\""); + if (0 != CaptiveProcess.Run(sc, "config \"" + controller.ServiceName + "\" binPath= \"" + escaped + "\"", cout.WriteLine, cout.WriteLine)) + { + cout.WriteLine("Could not reconfigure service path; ignoring."); + return; + } + + cout.WriteLine("Service binary path reconfigured successfully."); + } + + static int Start(ServiceController controller, TextWriter cout) + { + controller.Start(); + + if (controller.Status != ServiceControllerStatus.Running) + { + cout.WriteLine("Waiting up to 60 seconds for the service to start (currently: " + controller.Status + ")..."); + controller.WaitForStatus(ServiceControllerStatus.Running, TimeSpan.FromSeconds(60)); + } + + if (controller.Status == ServiceControllerStatus.Running) + { + cout.WriteLine("Started."); + return 0; + } + + cout.WriteLine("The service hasn't started successfully."); + return -1; + } + + [DllImport("shlwapi.dll")] + static extern bool PathIsNetworkPath(string pszPath); + + void Install(TextWriter cout) + { + cout.WriteLine("Installing service..."); + + if (PathIsNetworkPath(_storagePath.StorageRootPath)) + throw new ArgumentException("Seq requires a local (or SAN) storage location; network shares are not supported."); + + cout.WriteLine($"Updating the configuration in {_storagePath.ConfigFilePath}..."); + var config = SeqForwarderConfig.ReadOrInit(_storagePath.ConfigFilePath); + + if (!string.IsNullOrEmpty(_listenUri.ListenUri)) + { + config.Api.ListenUri = _listenUri.ListenUri; + SeqForwarderConfig.Write(_storagePath.ConfigFilePath, config); + } + + if (_serviceCredentials.IsUsernameSpecified) + { + if (!_serviceCredentials.IsPasswordSpecified) + throw new ArgumentException( + "If a service user account is specified, a password for the account must also be specified."); + + // https://technet.microsoft.com/en-us/library/cc794944(v=ws.10).aspx + cout.WriteLine($"Ensuring {_serviceCredentials.Username} is granted 'Log on as a Service' rights..."); + AccountRightsHelper.EnsureServiceLogOnRights(_serviceCredentials.Username); + } + + cout.WriteLine($"Granting {ServiceUsername} rights to {_storagePath.StorageRootPath}..."); + GiveFullControl(_storagePath.StorageRootPath); + + cout.WriteLine($"Granting {ServiceUsername} rights to {config.Diagnostics.InternalLogPath}..."); + GiveFullControl(config.Diagnostics.InternalLogPath); + + var listenUri = MakeListenUriReservationPattern(config.Api.ListenUri); + cout.WriteLine($"Adding URL reservation at {listenUri} for {ServiceUsername}..."); + var netshResult = CaptiveProcess.Run("netsh", $"http add urlacl url={listenUri} user=\"{ServiceUsername}\"", cout.WriteLine, cout.WriteLine); + if (netshResult != 0) + cout.WriteLine($"Could not add URL reservation for {listenUri}: `netsh` returned {netshResult}; ignoring"); + + var exePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory!, Program.BinaryName); + var forwarderRunCmdline = $"\"{exePath}\" run --storage=\"{_storagePath.StorageRootPath}\""; + + var binPath = forwarderRunCmdline.Replace("\"", "\\\""); + + var scCmdline = "create \"" + SeqForwarderWindowsService.WindowsServiceName + "\"" + + " binPath= \"" + binPath + "\"" + + " start= auto" + + " depend= Winmgmt/Tcpip/CryptSvc"; + + if (_serviceCredentials.IsUsernameSpecified) + scCmdline += $" obj= {_serviceCredentials.Username} password= {_serviceCredentials.Password}"; + + var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); + if (0 != CaptiveProcess.Run(sc, scCmdline, cout.WriteLine, cout.WriteLine)) + { + throw new ArgumentException("Service setup failed"); + } + + cout.WriteLine("Setting service restart policy..."); + if (0 != CaptiveProcess.Run(sc, $"failure \"{SeqForwarderWindowsService.WindowsServiceName}\" actions= restart/60000/restart/60000/restart/60000// reset= 600000", cout.WriteLine, cout.WriteLine)) + cout.WriteLine("Could not set service restart policy; ignoring"); + cout.WriteLine("Setting service description..."); + if (0 != CaptiveProcess.Run(sc, $"description \"{SeqForwarderWindowsService.WindowsServiceName}\" \"Durable storage and forwarding of application log events\"", cout.WriteLine, cout.WriteLine)) + cout.WriteLine("Could not set service description; ignoring"); + + cout.WriteLine("Service installed successfully."); + } + + void GiveFullControl(string target) + { + if (target == null) throw new ArgumentNullException(nameof(target)); + + if (!Directory.Exists(target)) + Directory.CreateDirectory(target); + + var storageInfo = new DirectoryInfo(target); + var storageAccessControl = storageInfo.GetAccessControl(); + storageAccessControl.AddAccessRule(new FileSystemAccessRule(ServiceUsername, + FileSystemRights.FullControl, AccessControlType.Allow)); + storageInfo.SetAccessControl(storageAccessControl); + } + + static string MakeListenUriReservationPattern(string uri) + { + var listenUri = uri.Replace("localhost", "+"); + if (!listenUri.EndsWith("/")) + listenUri += "/"; + return listenUri; + } + } +} + +#endif diff --git a/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs new file mode 100644 index 00000000..fb97172a --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs @@ -0,0 +1,83 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System; +using System.IO; +using System.ServiceProcess; +using Seq.Forwarder.ServiceProcess; + +// ReSharper disable UnusedType.Global + +namespace Seq.Forwarder.Cli.Commands +{ + [Command("forwarder", "restart", "Restart the Windows service")] + class RestartCommand : Command + { + protected override int Run(TextWriter cout) + { + try + { + var controller = new ServiceController(SeqForwarderWindowsService.WindowsServiceName); + + if (controller.Status != ServiceControllerStatus.Stopped) + { + cout.WriteLine("Stopping {0}...", controller.ServiceName); + controller.Stop(); + + if (controller.Status != ServiceControllerStatus.Stopped) + { + cout.WriteLine("Waiting up to 60 seconds for the service to stop (currently: " + + controller.Status + ")..."); + controller.WaitForStatus(ServiceControllerStatus.Stopped, TimeSpan.FromSeconds(60)); + } + + if (controller.Status != ServiceControllerStatus.Stopped) + { + cout.WriteLine("The service hasn't stopped successfully."); + return -1; + } + } + + cout.WriteLine("Starting {0}...", controller.ServiceName); + controller.Start(); + + if (controller.Status != ServiceControllerStatus.Running) + { + cout.WriteLine("Waiting up to 15 seconds for the service to start (currently: " + controller.Status + ")..."); + controller.WaitForStatus(ServiceControllerStatus.Running, TimeSpan.FromSeconds(15)); + } + + if (controller.Status == ServiceControllerStatus.Running) + { + cout.WriteLine("Started."); + return 0; + } + + cout.WriteLine("The service hasn't started successfully."); + return -1; + } + catch (Exception ex) + { + cout.WriteLine(ex.Message); + if (ex.InnerException != null) + cout.WriteLine(ex.InnerException.Message); + return 1; + } + } + } +} + +#endif diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs new file mode 100644 index 00000000..0af8e266 --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -0,0 +1,258 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Autofac; +using Seq.Forwarder.Config; +using Serilog; +using Serilog.Events; +using Serilog.Formatting.Compact; +using System; +using System.IO; +using System.Net; +using System.Threading.Tasks; +using Autofac.Extensions.DependencyInjection; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Hosting; +using Seq.Forwarder.Util; +using Seq.Forwarder.Web.Host; +using SeqCli.Cli; +using SeqCli.Cli.Features; +using Serilog.Core; + +// ReSharper disable UnusedType.Global + +namespace Seq.Forwarder.Cli.Commands +{ + [Command("forwarder", "run", "Run the server interactively")] + class RunCommand : Command + { + readonly StoragePathFeature _storagePath; + readonly ListenUriFeature _listenUri; + + bool _noLogo; + + public RunCommand() + { + Options.Add("nologo", _ => _noLogo = true); + _storagePath = Enable(); + _listenUri = Enable(); + } + + protected override async Task Run(string[] unrecognized) + { + if (Environment.UserInteractive) + { + if (!_noLogo) + { + WriteBanner(); + Console.WriteLine(); + } + + Console.WriteLine("Running as server; press Ctrl+C to exit."); + Console.WriteLine(); + } + + SeqForwarderConfig config; + + try + { + config = SeqForwarderConfig.ReadOrInit(_storagePath.ConfigFilePath); + } + catch (Exception ex) + { + await using var logger = CreateLogger( + LogEventLevel.Information, + SeqForwarderDiagnosticConfig.GetDefaultInternalLogPath()); + + logger.Fatal(ex, "Failed to load configuration from {ConfigFilePath}", _storagePath.ConfigFilePath); + return 1; + } + + Log.Logger = CreateLogger( + config.Diagnostics.InternalLoggingLevel, + config.Diagnostics.InternalLogPath, + config.Diagnostics.InternalLogServerUri, + config.Diagnostics.InternalLogServerApiKey); + + var listenUri = _listenUri.ListenUri ?? config.Api.ListenUri; + + try + { + ILifetimeScope? container = null; + using var host = new HostBuilder() + .UseSerilog() + .UseServiceProviderFactory(new AutofacServiceProviderFactory()) + .ConfigureContainer(builder => + { + builder.RegisterBuildCallback(ls => container = ls); + builder.RegisterModule(new SeqForwarderModule(_storagePath.BufferPath, config)); + }) + .ConfigureWebHostDefaults(web => + { + web.UseStartup(); + web.UseKestrel(options => + { + options.AddServerHeader = false; + options.AllowSynchronousIO = true; + }) + .ConfigureKestrel(options => + { + var apiListenUri = new Uri(listenUri); + + var ipAddress = apiListenUri.HostNameType switch + { + UriHostNameType.Basic => IPAddress.Any, + UriHostNameType.Dns => IPAddress.Any, + UriHostNameType.IPv4 => IPAddress.Parse(apiListenUri.Host), + UriHostNameType.IPv6 => IPAddress.Parse(apiListenUri.Host), + _ => throw new NotSupportedException($"Listen URI type `{apiListenUri.HostNameType}` is not supported.") + }; + + if (apiListenUri.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase)) + { + options.Listen(ipAddress, apiListenUri.Port, listenOptions => + { +#if WINDOWS + listenOptions.UseHttps(StoreName.My, apiListenUri.Host, + location: StoreLocation.LocalMachine, allowInvalid: true); +#else + listenOptions.UseHttps(); +#endif + }); + } + else + { + options.Listen(ipAddress, apiListenUri.Port); + } + }); + }) + .Build(); + + if (container == null) throw new Exception("Host did not build container."); + + var service = container.Resolve( + new TypedParameter(typeof(IHost), host), + new NamedParameter("listenUri", listenUri)); + + var exit = ExecutionEnvironment.SupportsStandardIO + ? RunStandardIO(service, Console.Out) + : RunService(service); + + return exit; + } + catch (Exception ex) + { + Log.Fatal(ex, "Unhandled exception"); + return -1; + } + finally + { + await Log.CloseAndFlushAsync(); + } + } + + static Logger CreateLogger( + LogEventLevel internalLoggingLevel, + string internalLogPath, + string? internalLogServerUri = null, + string? internalLogServerApiKey = null) + { + var loggerConfiguration = new LoggerConfiguration() + .Enrich.FromLogContext() + .Enrich.WithProperty("MachineName", Environment.MachineName) + .Enrich.WithProperty("Application", "Seq Forwarder") + .MinimumLevel.Is(internalLoggingLevel) + .MinimumLevel.Override("Microsoft", LogEventLevel.Warning) + .WriteTo.File( + new RenderedCompactJsonFormatter(), + GetRollingLogFilePathFormat(internalLogPath), + rollingInterval: RollingInterval.Day, + fileSizeLimitBytes: 1024 * 1024); + + if (Environment.UserInteractive) + loggerConfiguration.WriteTo.Console(restrictedToMinimumLevel: LogEventLevel.Information); + + if (!string.IsNullOrWhiteSpace(internalLogServerUri)) + loggerConfiguration.WriteTo.Seq( + internalLogServerUri, + apiKey: internalLogServerApiKey); + + return loggerConfiguration.CreateLogger(); + } + + static string GetRollingLogFilePathFormat(string internalLogPath) + { + if (internalLogPath == null) throw new ArgumentNullException(nameof(internalLogPath)); + + return Path.Combine(internalLogPath, "seq-forwarder-.log"); + } + + static int RunService(ServerService service) + { +#if WINDOWS + System.ServiceProcess.ServiceBase.Run([ + new ServiceProcess.SeqForwarderWindowsService(service) + ]); + return 0; +#else + throw new NotSupportedException("Windows services are not supported on this platform."); +#endif + } + + static int RunStandardIO(ServerService service, TextWriter cout) + { + service.Start(); + + try + { + Console.TreatControlCAsInput = true; + var k = Console.ReadKey(true); + while (k.Key != ConsoleKey.C || !k.Modifiers.HasFlag(ConsoleModifiers.Control)) + k = Console.ReadKey(true); + + cout.WriteLine("Ctrl+C pressed; stopping..."); + Console.TreatControlCAsInput = false; + } + catch (Exception ex) + { + Log.Debug(ex, "Console not attached, waiting for any input"); + Console.Read(); + } + + service.Stop(); + + return 0; + } + + static void WriteBanner() + { + Write("─", ConsoleColor.DarkGray, 47); + Console.WriteLine(); + Write(" Seq Forwarder", ConsoleColor.White); + Write(" ──", ConsoleColor.DarkGray); + Write(" © 2024 Datalust Pty Ltd", ConsoleColor.Gray); + Console.WriteLine(); + Write("─", ConsoleColor.DarkGray, 47); + Console.WriteLine(); + } + + static void Write(string s, ConsoleColor color, int repeats = 1) + { + Console.ForegroundColor = color; + for (var i = 0; i < repeats; ++i) + Console.Write(s); + Console.ResetColor(); + } + } +} diff --git a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs new file mode 100644 index 00000000..2b48ca4f --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs @@ -0,0 +1,67 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System; +using System.IO; +using System.ServiceProcess; +using Seq.Forwarder.ServiceProcess; + +namespace Seq.Forwarder.Cli.Commands +{ + [Command("forwarder", "start", "Start the Windows service")] + class StartCommand : Command + { + protected override int Run(TextWriter cout) + { + try + { + var controller = new ServiceController(SeqForwarderWindowsService.WindowsServiceName); + if (controller.Status != ServiceControllerStatus.Stopped) + { + cout.WriteLine("Cannot start {0}, current status is: {1}", controller.ServiceName, controller.Status); + return -1; + } + + cout.WriteLine("Starting {0}...", controller.ServiceName); + controller.Start(); + + if (controller.Status != ServiceControllerStatus.Running) + { + cout.WriteLine("Waiting up to 15 seconds for the service to start (currently: " + controller.Status + ")..."); + controller.WaitForStatus(ServiceControllerStatus.Running, TimeSpan.FromSeconds(15)); + } + + if (controller.Status == ServiceControllerStatus.Running) + { + cout.WriteLine("Started."); + return 0; + } + + cout.WriteLine("The service hasn't started successfully."); + return -1; + } + catch (Exception ex) + { + cout.WriteLine(ex.Message); + if (ex.InnerException != null) + cout.WriteLine(ex.InnerException.Message); + return -1; + } + } + } +} + +#endif diff --git a/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs new file mode 100644 index 00000000..c9261d93 --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs @@ -0,0 +1,51 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System; +using System.IO; +using System.ServiceProcess; +using Seq.Forwarder.ServiceProcess; + +namespace Seq.Forwarder.Cli.Commands +{ + [Command("forwarder", "status", "Show the status of the Seq Forwarder service")] + class StatusCommand : Command + { + protected override int Run(TextWriter cout) + { + try + { + var controller = new ServiceController(SeqForwarderWindowsService.WindowsServiceName); + cout.WriteLine("The Seq Forwarder service is installed and {0}.", controller.Status.ToString().ToLowerInvariant()); + } + catch (InvalidOperationException) + { + cout.WriteLine("The Seq Forwarder service is not installed."); + } + catch (Exception ex) + { + cout.WriteLine(ex.Message); + if (ex.InnerException != null) + cout.WriteLine(ex.InnerException.Message); + return 1; + } + + return 0; + } + } +} + +#endif diff --git a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs new file mode 100644 index 00000000..66c16637 --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs @@ -0,0 +1,68 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System; +using System.IO; +using System.ServiceProcess; +using Seq.Forwarder.ServiceProcess; + +namespace Seq.Forwarder.Cli.Commands +{ + [Command("forwarder", "stop", "Stop the Windows service")] + class StopCommand : Command + { + protected override int Run(TextWriter cout) + { + try + { + var controller = new ServiceController(SeqForwarderWindowsService.WindowsServiceName); + + if (controller.Status != ServiceControllerStatus.Running) + { + cout.WriteLine("Cannot stop {0}, current status is: {1}", controller.ServiceName, controller.Status); + return -1; + } + + cout.WriteLine("Stopping {0}...", controller.ServiceName); + controller.Stop(); + + if (controller.Status != ServiceControllerStatus.Stopped) + { + cout.WriteLine("Waiting up to 60 seconds for the service to stop (currently: " + controller.Status + ")..."); + controller.WaitForStatus(ServiceControllerStatus.Stopped, TimeSpan.FromSeconds(60)); + } + + if (controller.Status == ServiceControllerStatus.Stopped) + { + cout.WriteLine("Stopped."); + return 0; + } + + cout.WriteLine("The service hasn't stopped successfully."); + return -1; + } + catch (Exception ex) + { + cout.WriteLine(ex.Message); + if (ex.InnerException != null) + cout.WriteLine(ex.InnerException.Message); + return -1; + } + } + } +} + +#endif diff --git a/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs new file mode 100644 index 00000000..baf0320c --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs @@ -0,0 +1,51 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using System.Threading.Tasks; +using Seq.Forwarder.Multiplexing; +using SeqCli.Cli; +using SeqCli.Cli.Features; +using Serilog; + +namespace Seq.Forwarder.Cli.Commands +{ + [Command("forwarder", "truncate", "Clear the log buffer contents")] + class TruncateCommand : Command + { + readonly StoragePathFeature _storagePath; + + public TruncateCommand() + { + _storagePath = Enable(); + } + + protected override async Task Run(string[] args) + { + try + { + ActiveLogBufferMap.Truncate(_storagePath.BufferPath); + return 0; + } + catch (Exception ex) + { + await using var logger = new LoggerConfiguration().WriteTo.Console().CreateLogger(); + + logger.Fatal(ex, "Could not truncate log buffer"); + return 1; + } + } + } +} diff --git a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs new file mode 100644 index 00000000..ee717ab5 --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs @@ -0,0 +1,50 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System; +using System.IO; +using Seq.Forwarder.ServiceProcess; +using Seq.Forwarder.Util; + +namespace Seq.Forwarder.Cli.Commands +{ + [Command("forwarder", "uninstall", "Uninstall the Windows service")] + class UninstallCommand : Command + { + protected override int Run(TextWriter cout) + { + try + { + cout.WriteLine("Uninstalling service..."); + + var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); + var exitCode = CaptiveProcess.Run(sc, $"delete \"{SeqForwarderWindowsService.WindowsServiceName}\"", cout.WriteLine, cout.WriteLine); + if (exitCode != 0) + throw new InvalidOperationException($"The `sc.exe delete` call failed with exit code {exitCode}."); + + cout.WriteLine("Service uninstalled successfully."); + return 0; + } + catch (Exception ex) + { + cout.WriteLine("Could not uninstall the service: " + ex.Message); + return -1; + } + } + } +} + +#endif diff --git a/src/SeqCli/Cli/Features/ListenUriFeature.cs b/src/SeqCli/Cli/Features/ListenUriFeature.cs index 15c5352e..016543ee 100644 --- a/src/SeqCli/Cli/Features/ListenUriFeature.cs +++ b/src/SeqCli/Cli/Features/ListenUriFeature.cs @@ -7,7 +7,7 @@ class ListenUriFeature : CommandFeature public override void Enable(OptionSet options) { options.Add("l=|listen=", - "Set the listen Uri; http://localhost:15341/ is used by default.", + "Set the address `seqcli forwarder` will listen at; http://127.0.0.1:15341/ is used by default.", v => ListenUri = v); } } \ No newline at end of file diff --git a/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs b/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs new file mode 100644 index 00000000..7e7fcd1a --- /dev/null +++ b/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs @@ -0,0 +1,42 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using SeqCli.Cli; + +namespace Seq.Forwarder.Cli.Features +{ + class ServiceCredentialsFeature : CommandFeature + { + public bool IsUsernameSpecified => !string.IsNullOrEmpty(Username); + public bool IsPasswordSpecified => !string.IsNullOrEmpty(Password); + + public string Username { get; set; } = ""; + public string Password { get; set; } = ""; + + public override void Enable(OptionSet options) + { + options.Add("u=|username=", + "The name of a Windows account to run the service under; if not specified the Local System account will be used", + v => Username = v.Trim()); + + options.Add("p=|password=", + "The password for the Windows account to run the service under", + v => Password = v.Trim()); + } + } +} + +#endif diff --git a/src/SeqCli/Cli/Features/StoragePathFeature.cs b/src/SeqCli/Cli/Features/StoragePathFeature.cs index ce363a44..f706eb38 100644 --- a/src/SeqCli/Cli/Features/StoragePathFeature.cs +++ b/src/SeqCli/Cli/Features/StoragePathFeature.cs @@ -48,9 +48,9 @@ static string GetDefaultStorageRoot() static string? TryQueryInstalledStorageRoot() { #if WINDOWS - // if (Seq.Forwarder.Util.ServiceConfiguration.GetServiceStoragePath( - // Seq.Forwarder.ServiceProcess.SeqForwarderWindowsService.WindowsServiceName, out var storage)) - // return storage; + if (Seq.Forwarder.Util.ServiceConfiguration.GetServiceStoragePath( + Seq.Forwarder.ServiceProcess.SeqForwarderWindowsService.WindowsServiceName, out var storage)) + return storage; #endif return null; diff --git a/src/SeqCli/Forwarder/Config/SeqForwarderApiConfig.cs b/src/SeqCli/Forwarder/Config/SeqForwarderApiConfig.cs new file mode 100644 index 00000000..d2e0aaaa --- /dev/null +++ b/src/SeqCli/Forwarder/Config/SeqForwarderApiConfig.cs @@ -0,0 +1,21 @@ +// Copyright 2016-2017 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Seq.Forwarder.Config +{ + class SeqForwarderApiConfig + { + public string ListenUri { get; set; } = "http://localhost:15341"; + } +} diff --git a/src/SeqCli/Forwarder/Config/SeqForwarderConfig.cs b/src/SeqCli/Forwarder/Config/SeqForwarderConfig.cs new file mode 100644 index 00000000..d39abb59 --- /dev/null +++ b/src/SeqCli/Forwarder/Config/SeqForwarderConfig.cs @@ -0,0 +1,95 @@ +// Copyright 2016-2017 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using System.Linq; +using System.Reflection; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Serialization; + +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global + +namespace Seq.Forwarder.Config +{ + class SeqForwarderConfig + { + static JsonSerializerSettings SerializerSettings { get; } = new JsonSerializerSettings + { + ContractResolver = new CamelCasePropertyNamesContractResolver(), + Converters = + { + new StringEnumConverter() + } + }; + + public static SeqForwarderConfig ReadOrInit(string filename, bool includeEnvironmentVariables = true) + { + if (filename == null) throw new ArgumentNullException(nameof(filename)); + + if (!File.Exists(filename)) + { + var config = new SeqForwarderConfig(); + Write(filename, config); + return config; + } + + var content = File.ReadAllText(filename); + var combinedConfig = JsonConvert.DeserializeObject(content, SerializerSettings) + ?? throw new ArgumentException("Configuration content is null."); + + if (includeEnvironmentVariables) + { + // Any Environment Variables overwrite those in the Config File + var envVarConfig = new ConfigurationBuilder().AddEnvironmentVariables("FORWARDER_").Build(); + foreach (var sectionProperty in typeof(SeqForwarderConfig).GetTypeInfo().DeclaredProperties + .Where(p => p.GetMethod != null && p.GetMethod.IsPublic && !p.GetMethod.IsStatic)) + { + foreach (var subGroupProperty in sectionProperty.PropertyType.GetTypeInfo().DeclaredProperties + .Where(p => p.GetMethod != null && p.GetMethod.IsPublic && p.SetMethod != null && p.SetMethod.IsPublic && !p.GetMethod.IsStatic)) + { + var envVarName = sectionProperty.Name.ToUpper() + "_" + subGroupProperty.Name.ToUpper(); + var envVarVal = envVarConfig.GetValue(subGroupProperty.PropertyType, envVarName); + if (envVarVal != null) + { + subGroupProperty.SetValue(sectionProperty.GetValue(combinedConfig), envVarVal); + } + } + } + } + + return combinedConfig; + } + + public static void Write(string filename, SeqForwarderConfig data) + { + if (filename == null) throw new ArgumentNullException(nameof(filename)); + if (data == null) throw new ArgumentNullException(nameof(data)); + + var dir = Path.GetDirectoryName(filename); + if (!Directory.Exists(dir)) + Directory.CreateDirectory(dir!); + + var content = JsonConvert.SerializeObject(data, Formatting.Indented, SerializerSettings); + File.WriteAllText(filename, content); + } + + public SeqForwarderDiagnosticConfig Diagnostics { get; set; } = new SeqForwarderDiagnosticConfig(); + public SeqForwarderOutputConfig Output { get; set; } = new SeqForwarderOutputConfig(); + public SeqForwarderStorageConfig Storage { get; set; } = new SeqForwarderStorageConfig(); + public SeqForwarderApiConfig Api { get; set; } = new SeqForwarderApiConfig(); + } +} diff --git a/src/SeqCli/Forwarder/Config/SeqForwarderDiagnosticConfig.cs b/src/SeqCli/Forwarder/Config/SeqForwarderDiagnosticConfig.cs new file mode 100644 index 00000000..d1bca9f3 --- /dev/null +++ b/src/SeqCli/Forwarder/Config/SeqForwarderDiagnosticConfig.cs @@ -0,0 +1,44 @@ +// Copyright 2016-2017 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using Serilog.Events; + +namespace Seq.Forwarder.Config +{ + public class SeqForwarderDiagnosticConfig + { + public string InternalLogPath { get; set; } = GetDefaultInternalLogPath(); + public LogEventLevel InternalLoggingLevel { get; set; } = LogEventLevel.Information; + public string? InternalLogServerUri { get; set; } + public string? InternalLogServerApiKey { get; set; } + public bool IngestionLogShowDetail { get; set; } + + public static string GetDefaultInternalLogPath() + { + return Path.Combine( +#if WINDOWS + // Common, here, because the service may run as Local Service, which has no obvious home + // directory. + Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), +#else + // Specific to and writable by the current user. + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), +#endif + @"Seq", + "Logs"); + } + } +} diff --git a/src/SeqCli/Forwarder/Config/SeqForwarderOutputConfig.cs b/src/SeqCli/Forwarder/Config/SeqForwarderOutputConfig.cs new file mode 100644 index 00000000..a48bdf76 --- /dev/null +++ b/src/SeqCli/Forwarder/Config/SeqForwarderOutputConfig.cs @@ -0,0 +1,56 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using Newtonsoft.Json; +using Seq.Forwarder.Cryptography; + +// ReSharper disable UnusedMember.Global, AutoPropertyCanBeMadeGetOnly.Global + +namespace Seq.Forwarder.Config +{ + public class SeqForwarderOutputConfig + { + public string ServerUrl { get; set; } = "http://localhost:5341"; + public ulong EventBodyLimitBytes { get; set; } = 256 * 1024; + public ulong RawPayloadLimitBytes { get; set; } = 10 * 1024 * 1024; + public uint? PooledConnectionLifetimeMilliseconds { get; set; } = null; + + const string ProtectedDataPrefix = "pd."; + + public string? ApiKey { get; set; } + + public string? GetApiKey(IStringDataProtector dataProtector) + { + if (string.IsNullOrWhiteSpace(ApiKey)) + return null; + + if (!ApiKey.StartsWith(ProtectedDataPrefix)) + return ApiKey; + + return dataProtector.Unprotect(ApiKey.Substring(ProtectedDataPrefix.Length)); + } + + public void SetApiKey(string? apiKey, IStringDataProtector dataProtector) + { + if (string.IsNullOrWhiteSpace(apiKey)) + { + ApiKey = null; + return; + } + + ApiKey = $"{ProtectedDataPrefix}{dataProtector.Protect(apiKey)}"; + } + } +} diff --git a/src/SeqCli/Forwarder/Config/SeqForwarderStorageConfig.cs b/src/SeqCli/Forwarder/Config/SeqForwarderStorageConfig.cs new file mode 100644 index 00000000..2f713b7d --- /dev/null +++ b/src/SeqCli/Forwarder/Config/SeqForwarderStorageConfig.cs @@ -0,0 +1,21 @@ +// Copyright 2016-2017 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Seq.Forwarder.Config +{ + public class SeqForwarderStorageConfig + { + public ulong BufferSizeBytes { get; set; } = 64 * 1024 * 1024; + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs b/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs new file mode 100644 index 00000000..134eaaa3 --- /dev/null +++ b/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs @@ -0,0 +1,46 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System; +using System.Security.Cryptography; +using System.Text; + +namespace Seq.Forwarder.Cryptography +{ + class DpapiMachineScopeDataProtect : IStringDataProtector + { + public string Unprotect(string @protected) + { + var parts = @protected.Split(new[] { '$' }, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 2) + throw new InvalidOperationException("Encoded data format is invalid."); + + var bytes = Convert.FromBase64String(parts[0]); + var salt = Convert.FromBase64String(parts[1]); + var decoded = ProtectedData.Unprotect(bytes, salt, DataProtectionScope.LocalMachine); + return Encoding.UTF8.GetString(decoded); + } + + public string Protect(string value) + { + var salt = RandomNumberGenerator.GetBytes(16); + var bytes = ProtectedData.Protect(Encoding.UTF8.GetBytes(value), salt, DataProtectionScope.LocalMachine); + return $"{Convert.ToBase64String(bytes)}${Convert.ToBase64String(salt)}"; + } + } +} + +#endif diff --git a/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs b/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs new file mode 100644 index 00000000..24ef61b0 --- /dev/null +++ b/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs @@ -0,0 +1,8 @@ +namespace Seq.Forwarder.Cryptography +{ + public interface IStringDataProtector + { + string Protect(string value); + string Unprotect(string @protected); + } +} diff --git a/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs b/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs new file mode 100644 index 00000000..64ef755c --- /dev/null +++ b/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs @@ -0,0 +1,14 @@ +namespace Seq.Forwarder.Cryptography +{ + static class StringDataProtector + { + public static IStringDataProtector CreatePlatformDefault() + { +#if WINDOWS + return new DpapiMachineScopeDataProtect(); +#else + return new UnprotectedStringData(); +#endif + } + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs b/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs new file mode 100644 index 00000000..b5213375 --- /dev/null +++ b/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs @@ -0,0 +1,22 @@ +#if !WINDOWS + +using Serilog; + +namespace Seq.Forwarder.Cryptography +{ + public class UnprotectedStringData : IStringDataProtector + { + public string Protect(string value) + { + Log.Warning("Data protection is not available on this platform; sensitive values will be stored in plain text"); + return value; + } + + public string Unprotect(string @protected) + { + return @protected; + } + } +} + +#endif diff --git a/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs b/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs new file mode 100644 index 00000000..00797ec0 --- /dev/null +++ b/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs @@ -0,0 +1,49 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using Serilog.Core; +using Serilog.Events; + +namespace Seq.Forwarder.Diagnostics +{ + public class InMemorySink : ILogEventSink + { + readonly int _queueLength; + readonly ConcurrentQueue _queue = new ConcurrentQueue(); + + public InMemorySink(int queueLength) + { + _queueLength = queueLength; + } + + public IEnumerable Read() + { + return _queue.ToArray(); + } + + public void Emit(LogEvent logEvent) + { + if (logEvent == null) throw new ArgumentNullException(nameof(logEvent)); + _queue.Enqueue(logEvent); + + while (_queue.Count > _queueLength) + { + _queue.TryDequeue(out _); + } + } + } +} diff --git a/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs b/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs new file mode 100644 index 00000000..52e5af13 --- /dev/null +++ b/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs @@ -0,0 +1,65 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Net; +using Serilog; +using Serilog.Events; + +namespace Seq.Forwarder.Diagnostics +{ + static class IngestionLog + { + const int Capacity = 100; + + static readonly InMemorySink Sink = new InMemorySink(Capacity); + + public static ILogger Log { get; } + + static IngestionLog() + { + Log = new LoggerConfiguration() + .MinimumLevel.Verbose() + .WriteTo.Sink(Sink) + .WriteTo.Logger(Serilog.Log.Logger) + .CreateLogger(); + } + + public static IEnumerable Read() + { + return Sink.Read(); + } + + public static ILogger ForClient(IPAddress clientHostIP) + { + return Log.ForContext("ClientHostIP", clientHostIP); + } + + public static ILogger ForPayload(IPAddress clientHostIP, string payload) + { + var prefix = CapturePrefix(payload); + return ForClient(clientHostIP) + .ForContext("StartToLog", prefix.Length) + .ForContext("DocumentStart", prefix); + } + + static string CapturePrefix(string line) + { + if (line == null) throw new ArgumentNullException(nameof(line)); + var startToLog = Math.Min(line.Length, 1024); + return line.Substring(0, startToLog); + } + } +} diff --git a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs new file mode 100644 index 00000000..637f4c09 --- /dev/null +++ b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs @@ -0,0 +1,38 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using Seq.Forwarder.Shipper; +using Seq.Forwarder.Storage; + +namespace Seq.Forwarder.Multiplexing +{ + sealed class ActiveLogBuffer : IDisposable + { + public LogShipper Shipper { get; } + public LogBuffer Buffer { get; } + + public ActiveLogBuffer(LogBuffer logBuffer, LogShipper logShipper) + { + Buffer = logBuffer ?? throw new ArgumentNullException(nameof(logBuffer)); + Shipper = logShipper ?? throw new ArgumentNullException(nameof(logShipper)); + } + + public void Dispose() + { + Shipper.Dispose(); + Buffer.Dispose(); + } + } +} diff --git a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs new file mode 100644 index 00000000..13779570 --- /dev/null +++ b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs @@ -0,0 +1,235 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using Seq.Forwarder.Config; +using Seq.Forwarder.Cryptography; +using Seq.Forwarder.Storage; +using Seq.Forwarder.Web; +using Serilog; + +namespace Seq.Forwarder.Multiplexing +{ + public class ActiveLogBufferMap : IDisposable + { + const string DataFileName = "data.mdb", LockFileName = "lock.mdb", ApiKeyFileName = ".apikey"; + + readonly ulong _bufferSizeBytes; + readonly SeqForwarderOutputConfig _outputConfig; + readonly ILogShipperFactory _shipperFactory; + readonly IStringDataProtector _dataProtector; + readonly string _bufferPath; + readonly ILogger _log = Log.ForContext(); + + readonly object _sync = new object(); + bool _loaded; + ActiveLogBuffer? _noApiKeyLogBuffer; + readonly Dictionary _buffersByApiKey = new Dictionary(); + + public ActiveLogBufferMap( + string bufferPath, + SeqForwarderStorageConfig storageConfig, + SeqForwarderOutputConfig outputConfig, + ILogShipperFactory logShipperFactory, + IStringDataProtector dataProtector) + { + _bufferSizeBytes = storageConfig.BufferSizeBytes; + _outputConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); + _shipperFactory = logShipperFactory ?? throw new ArgumentNullException(nameof(logShipperFactory)); + _dataProtector = dataProtector ?? throw new ArgumentNullException(nameof(dataProtector)); + _bufferPath = bufferPath ?? throw new ArgumentNullException(nameof(bufferPath)); + } + + // The odd three-stage initialization improves our chances of correctly tearing down the `LightningEnvironment`s within + // `LogBuffer`s in the event of a failure during start-up. See: https://github.com/CoreyKaylor/Lightning.NET/blob/master/src/LightningDB/LightningEnvironment.cs#L252 + public void Load() + { + // At startup, we look for buffers and either delete them if they're empty, or load them + // up if they're not. This garbage collection at start-up is a simplification, + // we might try cleaning up in the background if the gains are worthwhile, although more synchronization + // would be required. + + lock (_sync) + { + if (_loaded) throw new InvalidOperationException("The log buffer map is already loaded."); + + Directory.CreateDirectory(_bufferPath); + + var defaultDataFilePath = Path.Combine(_bufferPath, DataFileName); + if (File.Exists(defaultDataFilePath)) + { + _log.Information("Loading the default log buffer in {Path}", _bufferPath); + var buffer = new LogBuffer(_bufferPath, _bufferSizeBytes); + if (buffer.Peek(0).Length == 0) + { + _log.Information("The default buffer is empty and will be removed until more data is received"); + buffer.Dispose(); + File.Delete(defaultDataFilePath); + var lockFilePath = Path.Combine(_bufferPath, LockFileName); + if (File.Exists(lockFilePath)) + File.Delete(lockFilePath); + } + else + { + _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _outputConfig.GetApiKey(_dataProtector))); + } + } + + foreach (var subfolder in Directory.GetDirectories(_bufferPath)) + { + var encodedApiKeyFilePath = Path.Combine(subfolder, ApiKeyFileName); + if (!File.Exists(encodedApiKeyFilePath)) + { + _log.Information("Folder {Path} does not appear to be a log buffer; skipping", subfolder); + continue; + } + + _log.Information("Loading an API-key specific buffer in {Path}", subfolder); + var apiKey = _dataProtector.Unprotect(File.ReadAllText(encodedApiKeyFilePath)); + + var buffer = new LogBuffer(subfolder, _bufferSizeBytes); + if (buffer.Peek(0).Length == 0) + { + _log.Information("API key-specific buffer in {Path} is empty and will be removed until more data is received", subfolder); + buffer.Dispose(); + Directory.Delete(subfolder, true); + } + else + { + var activeBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, apiKey)); + _buffersByApiKey.Add(apiKey, activeBuffer); + } + } + + _loaded = true; + } + } + + public void Start() + { + lock (_sync) + { + if (!_loaded) throw new InvalidOperationException("The log buffer map is not loaded."); + + foreach (var buffer in OpenBuffers) + { + buffer.Shipper.Start(); + } + } + } + + public void Stop() + { + lock (_sync) + { + // Hard to ensure _loaded is set in all cases, better here to be forgiving and + // permit a clean shut-down. + + foreach (var buffer in OpenBuffers) + { + buffer.Shipper.Stop(); + } + } + } + + public LogBuffer GetLogBuffer(string? apiKey) + { + lock (_sync) + { + if (!_loaded) throw new RequestProcessingException("The forwarder service is starting up.", HttpStatusCode.ServiceUnavailable); + + if (apiKey == null) + { + if (_noApiKeyLogBuffer == null) + { + _log.Information("Creating a new default log buffer in {Path}", _bufferPath); + var buffer = new LogBuffer(_bufferPath, _bufferSizeBytes); + _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _outputConfig.GetApiKey(_dataProtector))); + _noApiKeyLogBuffer.Shipper.Start(); + } + return _noApiKeyLogBuffer.Buffer; + } + + if (_buffersByApiKey.TryGetValue(apiKey, out var existing)) + return existing.Buffer; + + var subfolder = Path.Combine(_bufferPath, Guid.NewGuid().ToString("n")); + _log.Information("Creating a new API key-specific log buffer in {Path}", subfolder); + Directory.CreateDirectory(subfolder); + File.WriteAllText(Path.Combine(subfolder, ".apikey"), _dataProtector.Protect(apiKey)); + var newBuffer = new LogBuffer(subfolder, _bufferSizeBytes); + var newActiveBuffer = new ActiveLogBuffer(newBuffer, _shipperFactory.Create(newBuffer, apiKey)); + _buffersByApiKey.Add(apiKey, newActiveBuffer); + newActiveBuffer.Shipper.Start(); + return newBuffer; + } + } + + public void Dispose() + { + lock (_sync) + { + foreach (var buffer in OpenBuffers) + { + buffer.Dispose(); + } + } + } + + public void Enumerate(Action action) + { + if (action == null) throw new ArgumentNullException(nameof(action)); + + lock (_sync) + { + foreach (var buffer in OpenBuffers) + { + buffer.Buffer.Enumerate(action); + } + } + } + + public static void Truncate(string bufferPath) + { + DeleteIfExists(Path.Combine(bufferPath, DataFileName)); + DeleteIfExists(Path.Combine(bufferPath, LockFileName)); + foreach (var subdirectory in Directory.GetDirectories(bufferPath)) + { + if (File.Exists(Path.Combine(subdirectory, ApiKeyFileName))) + Directory.Delete(subdirectory, true); + } + } + + static void DeleteIfExists(string filePath) + { + if (File.Exists(filePath)) + File.Delete(filePath); + } + + IEnumerable OpenBuffers + { + get + { + if (_noApiKeyLogBuffer != null) + yield return _noApiKeyLogBuffer; + + foreach (var buffer in _buffersByApiKey.Values) + yield return buffer; + } + } + } +} diff --git a/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs new file mode 100644 index 00000000..4de973ad --- /dev/null +++ b/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs @@ -0,0 +1,41 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Net.Http; +using Seq.Forwarder.Config; +using Seq.Forwarder.Shipper; +using Seq.Forwarder.Storage; + +namespace Seq.Forwarder.Multiplexing +{ + class HttpLogShipperFactory : ILogShipperFactory + { + readonly HttpClient _outputHttpClient; + readonly ServerResponseProxy _serverResponseProxy; + readonly SeqForwarderOutputConfig _outputConfig; + + public HttpLogShipperFactory(ServerResponseProxy serverResponseProxy, SeqForwarderOutputConfig outputConfig, HttpClient outputHttpClient) + { + _outputHttpClient = outputHttpClient; + _serverResponseProxy = serverResponseProxy ?? throw new ArgumentNullException(nameof(serverResponseProxy)); + _outputConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); + } + + public LogShipper Create(LogBuffer logBuffer, string? apiKey) + { + return new HttpLogShipper(logBuffer, apiKey, _outputConfig, _serverResponseProxy, _outputHttpClient); + } + } +} diff --git a/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs new file mode 100644 index 00000000..554324de --- /dev/null +++ b/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs @@ -0,0 +1,24 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Seq.Forwarder.Shipper; +using Seq.Forwarder.Storage; + +namespace Seq.Forwarder.Multiplexing +{ + public interface ILogShipperFactory + { + LogShipper Create(LogBuffer logBuffer, string? apiKey); + } +} diff --git a/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs new file mode 100644 index 00000000..f0dd9e44 --- /dev/null +++ b/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs @@ -0,0 +1,27 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Seq.Forwarder.Shipper; +using Seq.Forwarder.Storage; + +namespace Seq.Forwarder.Multiplexing +{ + class InertLogShipperFactory : ILogShipperFactory + { + public LogShipper Create(LogBuffer logBuffer, string? apiKey) + { + return new InertLogShipper(); + } + } +} diff --git a/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs b/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs new file mode 100644 index 00000000..86ccc768 --- /dev/null +++ b/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs @@ -0,0 +1,52 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Collections.Generic; + +namespace Seq.Forwarder.Multiplexing +{ + public class ServerResponseProxy + { + const string EmptyResponse = "{}"; + + readonly object _syncRoot = new object(); + readonly Dictionary _lastResponseByApiKey = new Dictionary(); + string _lastNoApiKeyResponse = EmptyResponse; + + public void SuccessResponseReturned(string? apiKey, string response) + { + lock (_syncRoot) + { + if (apiKey == null) + _lastNoApiKeyResponse = response; + else + _lastResponseByApiKey[apiKey] = response; + } + } + + public string GetResponseText(string? apiKey) + { + lock (_syncRoot) + { + if (apiKey == null) + return _lastNoApiKeyResponse; + + if (_lastResponseByApiKey.TryGetValue(apiKey, out var response)) + return response; + + return EmptyResponse; + } + } + } +} diff --git a/src/SeqCli/Forwarder/Properties/AssemblyInfo.cs b/src/SeqCli/Forwarder/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..da681b10 --- /dev/null +++ b/src/SeqCli/Forwarder/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Seq.Forwarder.Tests")] diff --git a/src/SeqCli/Forwarder/Schema/EventSchema.cs b/src/SeqCli/Forwarder/Schema/EventSchema.cs new file mode 100644 index 00000000..aa6c6f6c --- /dev/null +++ b/src/SeqCli/Forwarder/Schema/EventSchema.cs @@ -0,0 +1,187 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Linq; +using Newtonsoft.Json.Linq; +using Serilog.Parsing; +using Seq.Forwarder.Util; + +namespace Seq.Forwarder.Schema +{ + static class EventSchema + { + static readonly MessageTemplateParser MessageTemplateParser = new MessageTemplateParser(); + + static readonly HashSet ClefReifiedProperties = new HashSet + { + "@t", "@m", "@mt", "@l", "@x", "@i", "@r" + }; + + public static bool FromClefFormat(in int lineNumber, JObject compactFormat, [MaybeNullWhen(false)] out JObject rawFormat, [MaybeNullWhen(true)] out string error) + { + var result = new JObject(); + + var rawTimestamp = compactFormat["@t"]; + if (rawTimestamp == null) + { + error = $"The event on line {lineNumber} does not carry an `@t` timestamp property."; + rawFormat = default; + return false; + } + + if (rawTimestamp.Type != JTokenType.String) + { + error = $"The event on line {lineNumber} has an invalid `@t` timestamp property; the value must be a JSON string."; + rawFormat = default; + return false; + } + + if (!DateTimeOffset.TryParse(rawTimestamp.Value(), out _)) + { + error = $"The timestamp value `{rawTimestamp}` on line {lineNumber} could not be parsed."; + rawFormat = default; + return false; + } + + result.Add("Timestamp", rawTimestamp); + + var properties = new JObject(); + foreach (var property in compactFormat.Properties()) + { + if (property.Name.StartsWith("@@")) + properties.Add(property.Name.Substring(1), property.Value); + else if (!ClefReifiedProperties.Contains(property.Name)) + properties.Add(property.Name, property.Value); + } + + var x = compactFormat["@x"]; + if (x != null) + { + if (x.Type != JTokenType.String) + { + error = $"The event on line {lineNumber} has a non-string `@x` exception property."; + rawFormat = default; + return false; + } + + result.Add("Exception", x); + } + + var l = compactFormat["@l"]; + if (l != null) + { + if (l.Type != JTokenType.String) + { + error = $"The event on line {lineNumber} has a non-string `@l` level property."; + rawFormat = default; + return false; + } + + result.Add("Level", l); + } + + string? message = null; + var m = compactFormat["@m"]; + if (m != null) + { + if (m.Type != JTokenType.String) + { + error = $"The event on line {lineNumber} has a non-string `@m` message property."; + rawFormat = default; + return false; + } + + message = m.Value(); + } + + string? messageTemplate = null; + var mt = compactFormat["@mt"]; + if (mt != null) + { + if (mt.Type != JTokenType.String) + { + error = $"The event on line {lineNumber} has a non-string `@mt` message template property."; + rawFormat = default; + return false; + } + + messageTemplate = mt.Value(); + } + + if (message != null) + { + result.Add("RenderedMessage", message); + } + else if (messageTemplate != null && compactFormat["@r"] is JArray renderingsArray) + { + var template = MessageTemplateParser.Parse(messageTemplate); + var withFormat = template.Tokens.OfType().Where(pt => pt.Format != null); + + // ReSharper disable once PossibleMultipleEnumeration + if (withFormat.Count() == renderingsArray.Count) + { + // ReSharper disable once PossibleMultipleEnumeration + var renderingsByProperty = withFormat + .Zip(renderingsArray, (p, j) => new { p.PropertyName, Format = p.Format!, Rendering = j.Value() }) + .GroupBy(p => p.PropertyName) + .ToDictionary(g => g.Key, g => g.ToDictionaryDistinct(p => p.Format, p => p.Rendering)); + + var renderings = new JObject(); + result.Add("Renderings", renderings); + + foreach (var (property, propertyRenderings) in renderingsByProperty) + { + var byFormat = new JArray(); + renderings.Add(property, byFormat); + + foreach (var (format, rendering) in propertyRenderings) + { + var element = new JObject {{"Format", format}, {"Rendering", rendering}}; + byFormat.Add(element); + } + } + } + } + + messageTemplate ??= message ?? "No template provided"; + result.Add("MessageTemplate", messageTemplate); + + var eventTypeToken = compactFormat["@i"]; + if (eventTypeToken != null) + { + if (eventTypeToken.Type == JTokenType.Integer) + { + result.Add("EventType", uint.Parse(eventTypeToken.Value()!)); + } + else if (eventTypeToken.Type == JTokenType.String) + { + if (uint.TryParse(eventTypeToken.Value(), NumberStyles.HexNumber, + CultureInfo.InvariantCulture, out var eventType)) + { + result.Add("EventType", eventType); + } + else + { + // Seq would calculate a hash value from the string, here. Forwarder will ignore that + // case and preserve the value in an `@i` property for now. + result.Add("@i", eventTypeToken); + } + } + else + { + error = $"The `@i` event type value on line {lineNumber} is not in a string or numeric format."; + rawFormat = default; + return false; + } + } + + if (properties.Count != 0) + result.Add("Properties", properties); + + rawFormat = result; + error = null; + return true; + } + } +} diff --git a/src/SeqCli/Forwarder/SeqForwarderModule.cs b/src/SeqCli/Forwarder/SeqForwarderModule.cs new file mode 100644 index 00000000..d8323510 --- /dev/null +++ b/src/SeqCli/Forwarder/SeqForwarderModule.cs @@ -0,0 +1,86 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Net.Http; +using System.Threading; +using Autofac; +using Seq.Forwarder.Config; +using Seq.Forwarder.Cryptography; +using Seq.Forwarder.Multiplexing; +using Seq.Forwarder.Web.Host; + +namespace Seq.Forwarder +{ + class SeqForwarderModule : Module + { + readonly string _bufferPath; + readonly SeqForwarderConfig _config; + + public SeqForwarderModule(string bufferPath, SeqForwarderConfig config) + { + _bufferPath = bufferPath ?? throw new ArgumentNullException(nameof(bufferPath)); + _config = config ?? throw new ArgumentNullException(nameof(config)); + } + + protected override void Load(ContainerBuilder builder) + { + builder.RegisterType().SingleInstance(); + builder.RegisterType() + .WithParameter("bufferPath", _bufferPath) + .SingleInstance(); + + builder.RegisterType().As(); + builder.RegisterType().SingleInstance(); + + builder.Register(c => + { + var outputConfig = c.Resolve(); + var baseUri = outputConfig.ServerUrl; + if (string.IsNullOrWhiteSpace(baseUri)) + throw new ArgumentException("The destination Seq server URL must be configured in SeqForwarder.json."); + + if (!baseUri.EndsWith("/")) + baseUri += "/"; + + // additional configuration options that require the use of SocketsHttpHandler should be added to + // this expression, using an "or" operator. + + var hasSocketHandlerOption = + (outputConfig.PooledConnectionLifetimeMilliseconds.HasValue); + + if (hasSocketHandlerOption) + { + var httpMessageHandler = new SocketsHttpHandler() + { + PooledConnectionLifetime = (outputConfig.PooledConnectionLifetimeMilliseconds.HasValue) ? TimeSpan.FromMilliseconds(outputConfig.PooledConnectionLifetimeMilliseconds.Value) : Timeout.InfiniteTimeSpan, + }; + + return new HttpClient(httpMessageHandler) { BaseAddress = new Uri(baseUri) }; + } + + return new HttpClient() { BaseAddress = new Uri(baseUri) }; + + }).SingleInstance(); + + builder.RegisterInstance(StringDataProtector.CreatePlatformDefault()); + + builder.RegisterInstance(_config); + builder.RegisterInstance(_config.Api); + builder.RegisterInstance(_config.Diagnostics); + builder.RegisterInstance(_config.Output); + builder.RegisterInstance(_config.Storage); + } + } +} diff --git a/src/SeqCli/Forwarder/ServiceProcess/SeqForwarderWindowsService.cs b/src/SeqCli/Forwarder/ServiceProcess/SeqForwarderWindowsService.cs new file mode 100644 index 00000000..013f67be --- /dev/null +++ b/src/SeqCli/Forwarder/ServiceProcess/SeqForwarderWindowsService.cs @@ -0,0 +1,52 @@ +// Copyright 2020 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System.Net; +using System.ServiceProcess; +using Seq.Forwarder.Web.Host; + +namespace Seq.Forwarder.ServiceProcess +{ + class SeqForwarderWindowsService : ServiceBase + { + readonly ServerService _serverService; + + public static string WindowsServiceName { get; } = "Seq Forwarder"; + + public SeqForwarderWindowsService(ServerService serverService) + { + // Enable TLS 1.2 Support. + // .NET Framework 4.5.2 does not have it enabled by default + ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; + + _serverService = serverService; + + ServiceName = WindowsServiceName; + } + + protected override void OnStart(string[] args) + { + _serverService.Start(); + } + + protected override void OnStop() + { + _serverService.Stop(); + } + } +} + +#endif diff --git a/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs b/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs new file mode 100644 index 00000000..84c32f6c --- /dev/null +++ b/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs @@ -0,0 +1,74 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace Seq.Forwarder.Shipper +{ + class ExponentialBackoffConnectionSchedule + { + static readonly TimeSpan MinimumBackoffPeriod = TimeSpan.FromSeconds(5); + static readonly TimeSpan MaximumBackoffInterval = TimeSpan.FromMinutes(10); + + readonly TimeSpan _period; + + int _failuresSinceSuccessfulConnection; + + public ExponentialBackoffConnectionSchedule(TimeSpan period) + { + if (period < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(period), "The connection retry period must be a positive timespan"); + + _period = period; + } + + public void MarkSuccess() + { + _failuresSinceSuccessfulConnection = 0; + } + + public void MarkFailure() + { + ++_failuresSinceSuccessfulConnection; + } + + public bool LastConnectionFailed => _failuresSinceSuccessfulConnection != 0; + + public TimeSpan NextInterval + { + get + { + // Available, and first failure, just try the batch interval + if (_failuresSinceSuccessfulConnection <= 1) return _period; + + // Second failure, start ramping up the interval - first 2x, then 4x, ... + var backoffFactor = Math.Pow(2, (_failuresSinceSuccessfulConnection - 1)); + + // If the period is ridiculously short, give it a boost so we get some + // visible backoff. + var backoffPeriod = Math.Max(_period.Ticks, MinimumBackoffPeriod.Ticks); + + // The "ideal" interval + var backedOff = (long)(backoffPeriod * backoffFactor); + + // Capped to the maximum interval + var cappedBackoff = Math.Min(MaximumBackoffInterval.Ticks, backedOff); + + // Unless that's shorter than the base interval, in which case we'll just apply the period + var actual = Math.Max(_period.Ticks, cappedBackoff); + + return TimeSpan.FromTicks(actual); + } + } + } +} diff --git a/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs b/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs new file mode 100644 index 00000000..5ebe7eb1 --- /dev/null +++ b/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs @@ -0,0 +1,251 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using Seq.Forwarder.Config; +using Seq.Forwarder.Storage; +using Serilog; +using System.Threading.Tasks; +using Seq.Forwarder.Multiplexing; +using Seq.Forwarder.Util; + +namespace Seq.Forwarder.Shipper +{ + sealed class HttpLogShipper : LogShipper + { + const string BulkUploadResource = "api/events/raw"; + + readonly string? _apiKey; + readonly LogBuffer _logBuffer; + readonly SeqForwarderOutputConfig _outputConfig; + readonly HttpClient _httpClient; + readonly ExponentialBackoffConnectionSchedule _connectionSchedule; + readonly ServerResponseProxy _serverResponseProxy; + DateTime _nextRequiredLevelCheck; + + readonly object _stateLock = new object(); + readonly Timer _timer; + bool _started; + + volatile bool _unloading; + + static readonly TimeSpan QuietWaitPeriod = TimeSpan.FromSeconds(2), MaximumConnectionInterval = TimeSpan.FromMinutes(2); + + public HttpLogShipper(LogBuffer logBuffer, string? apiKey, SeqForwarderOutputConfig outputConfig, ServerResponseProxy serverResponseProxy, HttpClient outputHttpClient) + { + _apiKey = apiKey; + _httpClient = outputHttpClient ?? throw new ArgumentNullException(nameof(outputHttpClient)); + _logBuffer = logBuffer ?? throw new ArgumentNullException(nameof(logBuffer)); + _outputConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); + _serverResponseProxy = serverResponseProxy ?? throw new ArgumentNullException(nameof(serverResponseProxy)); + _connectionSchedule = new ExponentialBackoffConnectionSchedule(QuietWaitPeriod); + _timer = new Timer(s => OnTick()); + } + + public override void Start() + { + lock (_stateLock) + { + if (_started) + throw new InvalidOperationException("The shipper has already started."); + + if (_unloading) + throw new InvalidOperationException("The shipper is unloading."); + + Log.Information("Log shipper started, events will be dispatched to {ServerUrl}", _outputConfig.ServerUrl); + + _nextRequiredLevelCheck = DateTime.UtcNow.Add(MaximumConnectionInterval); + _started = true; + SetTimer(); + } + } + + public override void Stop() + { + lock (_stateLock) + { + if (_unloading) + return; + + _unloading = true; + + if (!_started) + return; + } + + var wh = new ManualResetEvent(false); + if (_timer.Dispose(wh)) + wh.WaitOne(); + } + + public override void Dispose() + { + Stop(); + } + + void SetTimer() + { + _timer.Change(_connectionSchedule.NextInterval, Timeout.InfiniteTimeSpan); + } + + void OnTick() + { + OnTickAsync().Wait(); + } + + async Task OnTickAsync() + { + try + { + var sendingSingles = 0; + do + { + var available = _logBuffer.Peek((int)_outputConfig.RawPayloadLimitBytes); + if (available.Length == 0) + { + if (DateTime.UtcNow < _nextRequiredLevelCheck || _connectionSchedule.LastConnectionFailed) + { + // For whatever reason, there's nothing waiting to send. This means we should try connecting again at the + // regular interval, so mark the attempt as successful. + _connectionSchedule.MarkSuccess(); + break; + } + } + + MakePayload(available, sendingSingles > 0, out Stream payload, out ulong lastIncluded); + + var content = new StreamContent(new UnclosableStreamWrapper(payload)); + content.Headers.ContentType = new MediaTypeHeaderValue("application/json") + { + CharSet = Encoding.UTF8.WebName + }; + + if (_apiKey != null) + { + content.Headers.Add(SeqApi.ApiKeyHeaderName, _apiKey); + } + + var result = await _httpClient.PostAsync(BulkUploadResource, content); + if (result.IsSuccessStatusCode) + { + _connectionSchedule.MarkSuccess(); + _logBuffer.Dequeue(lastIncluded); + if (sendingSingles > 0) + sendingSingles--; + + _serverResponseProxy.SuccessResponseReturned(_apiKey, await result.Content.ReadAsStringAsync()); + _nextRequiredLevelCheck = DateTime.UtcNow.Add(MaximumConnectionInterval); + } + else if (result.StatusCode == HttpStatusCode.BadRequest || + result.StatusCode == HttpStatusCode.RequestEntityTooLarge) + { + // The connection attempt was successful - the payload we sent was the problem. + _connectionSchedule.MarkSuccess(); + + if (sendingSingles != 0) + { + payload.Position = 0; + var payloadText = await new StreamReader(payload, Encoding.UTF8).ReadToEndAsync(); + Log.Error("HTTP shipping failed with {StatusCode}: {Result}; payload was {InvalidPayload}", result.StatusCode, await result.Content.ReadAsStringAsync(), payloadText); + _logBuffer.Dequeue(lastIncluded); + sendingSingles = 0; + } + else + { + // Unscientific (should "binary search" in batches) but sending the next + // hundred events singly should flush out the problematic one. + sendingSingles = 100; + } + } + else + { + _connectionSchedule.MarkFailure(); + Log.Error("Received failed HTTP shipping result {StatusCode}: {Result}", result.StatusCode, await result.Content.ReadAsStringAsync()); + break; + } + } + while (true); + } + catch (HttpRequestException hex) + { + Log.Warning(hex, "HTTP request failed when sending a batch from the log shipper"); + _connectionSchedule.MarkFailure(); + } + catch (Exception ex) + { + Log.Error(ex, "Exception while sending a batch from the log shipper"); + _connectionSchedule.MarkFailure(); + } + finally + { + lock (_stateLock) + { + if (!_unloading) + SetTimer(); + } + } + } + + void MakePayload(LogBufferEntry[] entries, bool oneOnly, out Stream utf8Payload, out ulong lastIncluded) + { + if (entries == null) throw new ArgumentNullException(nameof(entries)); + lastIncluded = 0; + + var raw = new MemoryStream(); + var content = new StreamWriter(raw, Encoding.UTF8); + content.Write("{\"Events\":["); + content.Flush(); + var contentRemainingBytes = (int) _outputConfig.RawPayloadLimitBytes - 13; // Includes closing delims + + var delimStart = ""; + foreach (var logBufferEntry in entries) + { + if ((ulong)logBufferEntry.Value.Length > _outputConfig.EventBodyLimitBytes) + { + Log.Information("Oversized event will be skipped, {Payload}", Encoding.UTF8.GetString(logBufferEntry.Value)); + lastIncluded = logBufferEntry.Key; + continue; + } + + // lastIncluded indicates we've added at least one event + if (lastIncluded != 0 && contentRemainingBytes - (delimStart.Length + logBufferEntry.Value.Length) < 0) + break; + + content.Write(delimStart); + content.Flush(); + contentRemainingBytes -= delimStart.Length; + + raw.Write(logBufferEntry.Value, 0, logBufferEntry.Value.Length); + contentRemainingBytes -= logBufferEntry.Value.Length; + + lastIncluded = logBufferEntry.Key; + + delimStart = ","; + if (oneOnly) + break; + } + + content.Write("]}"); + content.Flush(); + raw.Position = 0; + utf8Payload = raw; + } + } +} diff --git a/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs b/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs new file mode 100644 index 00000000..164a2939 --- /dev/null +++ b/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs @@ -0,0 +1,31 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Seq.Forwarder.Shipper +{ + class InertLogShipper : LogShipper + { + public override void Start() + { + } + + public override void Stop() + { + } + + public override void Dispose() + { + } + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Shipper/LogShipper.cs b/src/SeqCli/Forwarder/Shipper/LogShipper.cs new file mode 100644 index 00000000..ac8f5157 --- /dev/null +++ b/src/SeqCli/Forwarder/Shipper/LogShipper.cs @@ -0,0 +1,25 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace Seq.Forwarder.Shipper +{ + public abstract class LogShipper : IDisposable + { + public abstract void Start(); + public abstract void Stop(); + public abstract void Dispose(); + } +} diff --git a/src/SeqCli/Forwarder/Shipper/SeqApi.cs b/src/SeqCli/Forwarder/Shipper/SeqApi.cs new file mode 100644 index 00000000..330dc3b8 --- /dev/null +++ b/src/SeqCli/Forwarder/Shipper/SeqApi.cs @@ -0,0 +1,21 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Seq.Forwarder.Shipper +{ + static class SeqApi + { + public const string ApiKeyHeaderName = "X-Seq-ApiKey"; + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Storage/LogBuffer.cs b/src/SeqCli/Forwarder/Storage/LogBuffer.cs new file mode 100644 index 00000000..bac905ea --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/LogBuffer.cs @@ -0,0 +1,280 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Serilog; + +namespace Seq.Forwarder.Storage +{ + public class LogBuffer : IDisposable + { + readonly ulong _bufferSizeBytes; + // readonly LightningEnvironment _env; + readonly object _sync = new object(); + bool _isDisposed; + ulong _nextId = 0, _entries = 0, _writtenSinceRotateCheck; + + public LogBuffer(string bufferPath, ulong bufferSizeBytes) + { + _bufferSizeBytes = bufferSizeBytes; + if (bufferPath == null) throw new ArgumentNullException(nameof(bufferPath)); + + // _env = new LightningEnvironment(bufferPath) + // { + // // Sparse; we'd hope fragmentation never gets this bad... + // MapSize = (long) bufferSizeBytes*10 + // }; + // + // _env.Open(); + // + // using (var tx = _env.BeginTransaction()) + // using (var db = tx.OpenDatabase()) + // { + // using (var cur = tx.CreateCursor(db)) + // { + // if (!cur.MoveToLast()) + // { + // _nextId = 1; + // } + // else + // { + // var current = cur.GetCurrent(); + // _nextId = ByteKeyToULongKey(current.Key) + 1; + // _entries = (ulong) tx.GetEntriesCount(db); + // } + // } + // } + + Log.Information("Log buffer open on {BufferPath}; {Entries} entries, next key will be {NextId}", bufferPath, _entries, _nextId); + } + + public void Dispose() + { + lock (_sync) + { + if (!_isDisposed) + { + _isDisposed = true; + // _env.Dispose(); + } + } + } + + public void Enqueue(byte[][] values) + { + if (values == null) throw new ArgumentNullException(nameof(values)); + + lock (_sync) + { + RequireNotDisposed(); + + // var totalPayloadWritten = 0UL; + // + // using (var tx = _env.BeginTransaction()) + // using (var db = tx.OpenDatabase()) + // { + // foreach (var v in values) + // { + // if (v == null) throw new ArgumentException("Value array may not contain null."); + // + // tx.Put(db, ULongKeyToByteKey(_nextId++), v); + // totalPayloadWritten += (ulong) v.Length; + // } + // + // tx.Commit(); + // _entries += (ulong) values.Length; + // _writtenSinceRotateCheck += totalPayloadWritten; + // } + + RotateIfRequired(); + } + } + + void RotateIfRequired() + { + if (_writtenSinceRotateCheck < _bufferSizeBytes/10) + return; + + _writtenSinceRotateCheck = 0; + // + // using (var tx = _env.BeginTransaction()) + // using (var db = tx.OpenDatabase()) + // { + // int err; + // if (0 != (err = Lmdb.mdb_env_info(_env.Handle(), out var estat))) + // throw new Exception(Lmdb.mdb_strerror(err)); + // + // MDBStat stat; + // if (0 != (err = Lmdb.mdb_stat(tx.Handle(), db.Handle(), out stat))) + // throw new Exception(Lmdb.mdb_strerror(err)); + // + // // http://www.openldap.org/lists/openldap-technical/201303/msg00145.html + // // 1) MDB_stat gives you the page size. + // // 2) MDB_envinfo tells the mapsize and the last_pgno.If you divide mapsize + // // by pagesize you'll get max pgno. The MAP_FULL error is returned when last_pgno reaches max pgno. + // + // var targetPages = _bufferSizeBytes/stat.ms_psize; + // if ((ulong) estat.me_last_pgno < targetPages && (double) (ulong) estat.me_last_pgno/targetPages < 0.75) + // return; + // + // var count = tx.GetEntriesCount(db); + // if (count == 0) + // { + // Log.Warning("Attempting to rotate buffer but no events are present"); + // return; + // } + // + // var toPurge = Math.Max(count / 4, 1); + // Log.Warning("Buffer is full; dropping {ToPurge} events to make room for new ones", + // toPurge); + // + // using (var cur = tx.CreateCursor(db)) + // { + // cur.MoveToFirst(); + // + // for (var i = 0; i < toPurge; ++i) + // { + // cur.Delete(); + // cur.MoveNext(); + // } + // } + // + // tx.Commit(); + // } + } + + public LogBufferEntry[] Peek(int maxValueBytesHint) + { + lock (_sync) + { + RequireNotDisposed(); + + var entries = new List(); + // + // using (var tx = _env.BeginTransaction(TransactionBeginFlags.ReadOnly)) + // using (var db = tx.OpenDatabase()) + // { + // using (var cur = tx.CreateCursor(db)) + // { + // if (cur.MoveToFirst()) + // { + // var entriesBytes = 0; + // + // do + // { + // var current = cur.GetCurrent(); + // var entry = new LogBufferEntry + // { + // Key = ByteKeyToULongKey(current.Key), + // Value = current.Value + // }; + // + // entriesBytes += entry.Value.Length; + // if (entries.Count != 0 && entriesBytes > maxValueBytesHint) + // break; + // + // entries.Add(entry); + // + // } while (cur.MoveNext()); + // } + // } + // } + + return entries.ToArray(); + } + } + + public void Dequeue(ulong toKey) + { + lock (_sync) + { + RequireNotDisposed(); + + // ulong deleted = 0; + // + // using (var tx = _env.BeginTransaction()) + // using (var db = tx.OpenDatabase()) + // { + // using (var cur = tx.CreateCursor(db)) + // { + // if (cur.MoveToFirst()) + // { + // do + // { + // var current = cur.GetCurrent(); + // if (ByteKeyToULongKey(current.Key) > toKey) + // break; + // + // cur.Delete(); + // deleted++; + // } while (cur.MoveNext()); + // } + // } + // + // tx.Commit(); + // _entries -= deleted; + // } + } + } + + void RequireNotDisposed() + { + if (_isDisposed) + throw new ObjectDisposedException(typeof(LogBuffer).FullName); + } + + static ulong ByteKeyToULongKey(byte[] key) + { + var copy = new byte[key.Length]; + for (var i = 0; i < key.Length; ++i) + copy[copy.Length - (i + 1)] = key[i]; + + return BitConverter.ToUInt64(copy, 0); + } + + static byte[] ULongKeyToByteKey(ulong key) + { + var k = BitConverter.GetBytes(key); + Array.Reverse(k); + return k; + } + + public void Enumerate(Action action) + { + if (action == null) throw new ArgumentNullException(nameof(action)); + + lock (_sync) + { + RequireNotDisposed(); + + // using (var tx = _env.BeginTransaction(TransactionBeginFlags.ReadOnly)) + // using (var db = tx.OpenDatabase()) + // { + // using (var cur = tx.CreateCursor(db)) + // { + // if (cur.MoveToFirst()) + // { + // do + // { + // var current = cur.GetCurrent(); + // action(ByteKeyToULongKey(current.Key), current.Value); + // } while (cur.MoveNext()); + // } + // } + // } + } + } + } +} diff --git a/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs b/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs new file mode 100644 index 00000000..464a7175 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs @@ -0,0 +1,24 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ReSharper disable InconsistentNaming + +namespace Seq.Forwarder.Storage +{ + public struct LogBufferEntry + { + public ulong Key; + public byte[] Value; + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Util/AccountRightsHelper.cs b/src/SeqCli/Forwarder/Util/AccountRightsHelper.cs new file mode 100644 index 00000000..6b73caf8 --- /dev/null +++ b/src/SeqCli/Forwarder/Util/AccountRightsHelper.cs @@ -0,0 +1,193 @@ +// Original interop code copyright Corinna John +// Used under CPOL. http://www.codeproject.com/info/cpol10.aspx +// http://www.codeproject.com/Articles/4863/LSA-Functions-Privileges-and-Impersonation +// Modified and reformatted. + +#if WINDOWS + +using System; +using System.Runtime.InteropServices; +using System.Text; + +// ReSharper disable FieldCanBeMadeReadOnly.Local + +namespace Seq.Forwarder.Util +{ + public static class AccountRightsHelper + { + [DllImport("advapi32.dll", PreserveSig = true)] + private static extern UInt32 LsaOpenPolicy( + ref LSA_UNICODE_STRING SystemName, + ref LSA_OBJECT_ATTRIBUTES ObjectAttributes, + Int32 DesiredAccess, + out IntPtr PolicyHandle + ); + + [DllImport("advapi32.dll", SetLastError = true, PreserveSig = true)] + private static extern long LsaAddAccountRights( + IntPtr PolicyHandle, + IntPtr AccountSid, + LSA_UNICODE_STRING[] UserRights, + long CountOfRights); + + [DllImport("advapi32")] + public static extern void FreeSid(IntPtr pSid); + + [DllImport("advapi32.dll", CharSet = CharSet.Auto, SetLastError = true, PreserveSig = true)] + private static extern bool LookupAccountName( + string lpSystemName, string lpAccountName, + IntPtr psid, + ref int cbsid, + StringBuilder domainName, ref int cbdomainLength, ref int use); + + [DllImport("advapi32.dll")] + private static extern long LsaClose(IntPtr ObjectHandle); + + [DllImport("kernel32.dll")] + private static extern int GetLastError(); + + [DllImport("advapi32.dll")] + private static extern long LsaNtStatusToWinError(long status); + + // define the structures + + [StructLayout(LayoutKind.Sequential)] + private struct LSA_UNICODE_STRING + { + public UInt16 Length; + public UInt16 MaximumLength; + public IntPtr Buffer; + } + + [StructLayout(LayoutKind.Sequential)] + private struct LSA_OBJECT_ATTRIBUTES + { + public int Length; + public IntPtr RootDirectory; + public LSA_UNICODE_STRING ObjectName; + public UInt32 Attributes; + public IntPtr SecurityDescriptor; + public IntPtr SecurityQualityOfService; + } + + // enum all policies + + [Flags] + private enum LSA_AccessPolicy : long + { + POLICY_VIEW_LOCAL_INFORMATION = 0x00000001L, + POLICY_VIEW_AUDIT_INFORMATION = 0x00000002L, + POLICY_GET_PRIVATE_INFORMATION = 0x00000004L, + POLICY_TRUST_ADMIN = 0x00000008L, + POLICY_CREATE_ACCOUNT = 0x00000010L, + POLICY_CREATE_SECRET = 0x00000020L, + POLICY_CREATE_PRIVILEGE = 0x00000040L, + POLICY_SET_DEFAULT_QUOTA_LIMITS = 0x00000080L, + POLICY_SET_AUDIT_REQUIREMENTS = 0x00000100L, + POLICY_AUDIT_LOG_ADMIN = 0x00000200L, + POLICY_SERVER_ADMIN = 0x00000400L, + POLICY_LOOKUP_NAMES = 0x00000800L, + POLICY_NOTIFICATION = 0x00001000L + } + + /// Adds a privilege to an account + /// Name of an account - "domain\account" or only "account" + /// Name ofthe privilege + /// The windows error code returned by LsaAddAccountRights + static long SetRight(string accountName, string privilegeName) + { + long winErrorCode; //contains the last error + + //pointer an size for the SID + IntPtr sid = IntPtr.Zero; + int sidSize = 0; + //StringBuilder and size for the domain name + StringBuilder domainName = new StringBuilder(); + int nameSize = 0; + //account-type variable for lookup + int accountType = 0; + + //get required buffer size + LookupAccountName(String.Empty, accountName, sid, ref sidSize, domainName, ref nameSize, ref accountType); + + //allocate buffers + domainName = new StringBuilder(nameSize); + sid = Marshal.AllocHGlobal(sidSize); + + //lookup the SID for the account + bool result = LookupAccountName(String.Empty, accountName, sid, ref sidSize, domainName, ref nameSize, ref accountType); + + if (!result) + { + winErrorCode = GetLastError(); + } + else + { + //initialize an empty unicode-string + LSA_UNICODE_STRING systemName = new LSA_UNICODE_STRING(); + //combine all policies + int access = (int)( + LSA_AccessPolicy.POLICY_AUDIT_LOG_ADMIN | + LSA_AccessPolicy.POLICY_CREATE_ACCOUNT | + LSA_AccessPolicy.POLICY_CREATE_PRIVILEGE | + LSA_AccessPolicy.POLICY_CREATE_SECRET | + LSA_AccessPolicy.POLICY_GET_PRIVATE_INFORMATION | + LSA_AccessPolicy.POLICY_LOOKUP_NAMES | + LSA_AccessPolicy.POLICY_NOTIFICATION | + LSA_AccessPolicy.POLICY_SERVER_ADMIN | + LSA_AccessPolicy.POLICY_SET_AUDIT_REQUIREMENTS | + LSA_AccessPolicy.POLICY_SET_DEFAULT_QUOTA_LIMITS | + LSA_AccessPolicy.POLICY_TRUST_ADMIN | + LSA_AccessPolicy.POLICY_VIEW_AUDIT_INFORMATION | + LSA_AccessPolicy.POLICY_VIEW_LOCAL_INFORMATION + ); + + //initialize a pointer for the policy handle + IntPtr policyHandle; + + //these attributes are not used, but LsaOpenPolicy wants them to exists + LSA_OBJECT_ATTRIBUTES ObjectAttributes = new LSA_OBJECT_ATTRIBUTES(); + ObjectAttributes.Length = 0; + ObjectAttributes.RootDirectory = IntPtr.Zero; + ObjectAttributes.Attributes = 0; + ObjectAttributes.SecurityDescriptor = IntPtr.Zero; + ObjectAttributes.SecurityQualityOfService = IntPtr.Zero; + + //get a policy handle + uint resultPolicy = LsaOpenPolicy(ref systemName, ref ObjectAttributes, access, out policyHandle); + winErrorCode = LsaNtStatusToWinError(resultPolicy); + + if (winErrorCode == 0) + { + //Now that we have the SID an the policy, + //we can add rights to the account. + + //initialize an unicode-string for the privilege name + LSA_UNICODE_STRING[] userRights = new LSA_UNICODE_STRING[1]; + userRights[0] = new LSA_UNICODE_STRING(); + userRights[0].Buffer = Marshal.StringToHGlobalUni(privilegeName); + userRights[0].Length = (UInt16)(privilegeName.Length * UnicodeEncoding.CharSize); + userRights[0].MaximumLength = (UInt16)((privilegeName.Length + 1) * UnicodeEncoding.CharSize); + + //add the right to the account + long res = LsaAddAccountRights(policyHandle, sid, userRights, 1); + winErrorCode = LsaNtStatusToWinError(res); + LsaClose(policyHandle); + } + + FreeSid(sid); + } + + return winErrorCode; + } + + public static void EnsureServiceLogOnRights(string accountName) + { + var err = SetRight(accountName, "SeServiceLogonRight"); + if (err != 0) + Marshal.ThrowExceptionForHR(Marshal.GetHRForLastWin32Error()); + } + } +} + +#endif diff --git a/src/SeqCli/Forwarder/Util/CaptiveProcess.cs b/src/SeqCli/Forwarder/Util/CaptiveProcess.cs new file mode 100644 index 00000000..dc12482e --- /dev/null +++ b/src/SeqCli/Forwarder/Util/CaptiveProcess.cs @@ -0,0 +1,82 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics; +using System.Threading; + +namespace Seq.Forwarder.Util +{ + public static class CaptiveProcess + { + public static int Run( + string fullExePath, + string? args = null, + Action? writeStdout = null, + Action? writeStderr = null, + string? workingDirectory = null) + { + if (fullExePath == null) throw new ArgumentNullException(nameof(fullExePath)); + + args ??= ""; + writeStdout ??= delegate { }; + writeStderr ??= delegate { }; + + var startInfo = new ProcessStartInfo + { + UseShellExecute = false, + RedirectStandardError = true, + RedirectStandardOutput = true, + WindowStyle = ProcessWindowStyle.Hidden, + CreateNoWindow = true, + ErrorDialog = false, + FileName = fullExePath, + Arguments = args + }; + + if (!string.IsNullOrEmpty(workingDirectory)) + startInfo.WorkingDirectory = workingDirectory; + + using var process = Process.Start(startInfo)!; + using var outputComplete = new ManualResetEvent(false); + using var errorComplete = new ManualResetEvent(false); + // ReSharper disable AccessToDisposedClosure + + process.OutputDataReceived += (_, e) => + { + if (e.Data == null) + outputComplete.Set(); + else + writeStdout(e.Data); + }; + process.BeginOutputReadLine(); + + process.ErrorDataReceived += (_, e) => + { + if (e.Data == null) + errorComplete.Set(); + else + writeStderr(e.Data); + }; + process.BeginErrorReadLine(); + + process.WaitForExit(); + + outputComplete.WaitOne(); + errorComplete.WaitOne(); + + return process.ExitCode; + } + } +} diff --git a/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs b/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs new file mode 100644 index 00000000..612bc684 --- /dev/null +++ b/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; + +namespace Seq.Forwarder.Util +{ + static class EnumerableExtensions + { + public static Dictionary ToDictionaryDistinct( + this IEnumerable enumerable, Func keySelector, Func valueSelector) + where TKey: notnull + { + var result = new Dictionary(); + foreach (var e in enumerable) + { + result[keySelector(e)] = valueSelector(e); + } + return result; + } + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs b/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs new file mode 100644 index 00000000..9e94295a --- /dev/null +++ b/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs @@ -0,0 +1,20 @@ +namespace Seq.Forwarder.Util +{ + static class ExecutionEnvironment + { + public static bool SupportsStandardIO => !IsRunningAsWindowsService; + + static bool IsRunningAsWindowsService + { + get + { +#if WINDOWS + var parent = WindowsProcess.GetParentProcess(); + return parent?.ProcessName == "services"; +#else + return false; +#endif + } + } + } +} diff --git a/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs b/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs new file mode 100644 index 00000000..2fda1fd5 --- /dev/null +++ b/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs @@ -0,0 +1,111 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.ServiceProcess; +using System.Text; + +namespace Seq.Forwarder.Util +{ + public static class ServiceConfiguration + { + public static bool GetServiceBinaryPath(ServiceController controller, TextWriter cout, [MaybeNullWhen(false)] out string path) + { + var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); + + var config = new StringBuilder(); + if (0 != CaptiveProcess.Run(sc, "qc \"" + controller.ServiceName + "\"", l => config.AppendLine(l), cout.WriteLine)) + { + cout.WriteLine("Could not query service path; ignoring."); + path = null; + return false; + } + + var lines = config.ToString() + .Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries) + .Select(l => l.Trim()); + + var line = lines + .SingleOrDefault(l => l.StartsWith("BINARY_PATH_NAME : ")); + + if (line == null) + { + cout.WriteLine("No existing binary path could be determined."); + path = null; + return false; + } + + path = line.Replace("BINARY_PATH_NAME : ", ""); + return true; + } + + static bool GetServiceCommandLine(string serviceName, TextWriter cout, [MaybeNullWhen(false)] out string path) + { + if (serviceName == null) throw new ArgumentNullException(nameof(serviceName)); + if (cout == null) throw new ArgumentNullException(nameof(cout)); + + var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); + + var config = new StringBuilder(); + if (0 != CaptiveProcess.Run(sc, "qc \"" + serviceName + "\"", l => config.AppendLine(l), cout.WriteLine)) + { + cout.WriteLine("Could not query service path; ignoring."); + path = null; + return false; + } + + var lines = config.ToString() + .Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries) + .Select(l => l.Trim()); + + var line = lines + .SingleOrDefault(l => l.StartsWith("BINARY_PATH_NAME : ")); + + if (line == null) + { + cout.WriteLine("No existing binary path could be determined."); + path = null; + return false; + } + + path = line.Replace("BINARY_PATH_NAME : ", ""); + return true; + } + + public static bool GetServiceStoragePath(string serviceName, out string? storage) + { + if (serviceName == null) throw new ArgumentNullException(nameof(serviceName)); + + if (GetServiceCommandLine(serviceName, new StringWriter(), out var binpath) && + binpath.Contains("--storage=\"")) + { + var start = binpath.IndexOf("--storage=\"", StringComparison.Ordinal) + 11; + var chop = binpath.Substring(start); + storage = chop.Substring(0, chop.IndexOf('"')); + return true; + } + + storage = null; + return false; + } + } +} + +#endif + diff --git a/src/SeqCli/Forwarder/Util/UnclosableStreamWrapper.cs b/src/SeqCli/Forwarder/Util/UnclosableStreamWrapper.cs new file mode 100644 index 00000000..ce86ea12 --- /dev/null +++ b/src/SeqCli/Forwarder/Util/UnclosableStreamWrapper.cs @@ -0,0 +1,60 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; + +namespace Seq.Forwarder.Util +{ + class UnclosableStreamWrapper : Stream + { + readonly Stream _stream; + + public UnclosableStreamWrapper(Stream stream) + { + _stream = stream ?? throw new ArgumentNullException(nameof(stream)); + } + + public override void Flush() + { + _stream.Flush(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + return _stream.Seek(offset, origin); + } + + public override void SetLength(long value) + { + _stream.SetLength(value); + } + + public override int Read(byte[] buffer, int offset, int count) + { + return _stream.Read(buffer, offset, count); + } + + public override void Write(byte[] buffer, int offset, int count) + { + _stream.Write(buffer, offset, count); + } + + public override bool CanRead => _stream.CanRead; + public override bool CanSeek => _stream.CanSeek; + public override bool CanWrite => _stream.CanWrite; + public override long Length => _stream.Length; + public override long Position { get { return _stream.Position; } set { _stream.Position = value; } } + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Util/WindowsProcess.cs b/src/SeqCli/Forwarder/Util/WindowsProcess.cs new file mode 100644 index 00000000..98a20930 --- /dev/null +++ b/src/SeqCli/Forwarder/Util/WindowsProcess.cs @@ -0,0 +1,51 @@ +#if WINDOWS + +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.Runtime.InteropServices; +using Serilog; + +// ReSharper disable once InconsistentNaming + +namespace Seq.Forwarder.Util +{ + static class WindowsProcess + { + [StructLayout(LayoutKind.Sequential)] + readonly struct PROCESS_BASIC_INFORMATION + { + readonly IntPtr _reserved1; + readonly IntPtr _pebBaseAddress; + readonly IntPtr _reserved2_0; + readonly IntPtr _reserved2_1; + readonly IntPtr _uniqueProcessId; + public readonly IntPtr InheritedFromUniqueProcessId; + } + + [DllImport("ntdll.dll")] + static extern int NtQueryInformationProcess(IntPtr processHandle, int processInformationClass, ref PROCESS_BASIC_INFORMATION processInformation, int processInformationLength, out int returnLength); + + public static Process? GetParentProcess() + { + var currentProcess = Process.GetCurrentProcess(); + + var pbi = new PROCESS_BASIC_INFORMATION(); + var status = NtQueryInformationProcess(currentProcess.Handle, 0, ref pbi, Marshal.SizeOf(pbi), out _); + if (status != 0) + throw new Win32Exception(status); + + try + { + return Process.GetProcessById(pbi.InheritedFromUniqueProcessId.ToInt32()); + } + catch (Exception ex) + { + Log.Debug(ex, "Could not query parent process information"); + return null; + } + } + } +} + +#endif diff --git a/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs b/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs new file mode 100644 index 00000000..e688faac --- /dev/null +++ b/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs @@ -0,0 +1,57 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.IO; +using System.Text; +using Microsoft.AspNetCore.Mvc; +using Seq.Forwarder.Config; +using Seq.Forwarder.Diagnostics; +using Serilog.Formatting.Display; + +namespace Seq.Forwarder.Web.Api +{ + public class ApiRootController : Controller + { + static readonly Encoding Encoding = new UTF8Encoding(false); + readonly MessageTemplateTextFormatter _ingestionLogFormatter; + + public ApiRootController(SeqForwarderDiagnosticConfig diagnosticConfig) + { + var template = "[{Timestamp:o} {Level:u3}] {Message}{NewLine}"; + if (diagnosticConfig.IngestionLogShowDetail) + template += "Client IP address: {ClientHostIP}{NewLine}First {StartToLog} characters of payload: {DocumentStart:l}{NewLine}{Exception}{NewLine}"; + + _ingestionLogFormatter = new MessageTemplateTextFormatter(template); + } + + [HttpGet, Route("")] + public IActionResult Index() + { + var events = IngestionLog.Read(); + using var log = new StringWriter(); + foreach (var logEvent in events) + { + _ingestionLogFormatter.Format(logEvent, log); + } + + return Content(log.ToString(), "text/plain", Encoding); + } + + [HttpGet, Route("api")] + public IActionResult Resources() + { + return Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Encoding); + } + } +} diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionController.cs b/src/SeqCli/Forwarder/Web/Api/IngestionController.cs new file mode 100644 index 00000000..76fbbc11 --- /dev/null +++ b/src/SeqCli/Forwarder/Web/Api/IngestionController.cs @@ -0,0 +1,246 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Net.Http.Headers; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Seq.Forwarder.Config; +using Seq.Forwarder.Diagnostics; +using Seq.Forwarder.Multiplexing; +using Seq.Forwarder.Schema; +using Seq.Forwarder.Shipper; + +namespace Seq.Forwarder.Web.Api +{ + public class IngestionController : Controller + { + static readonly Encoding Encoding = new UTF8Encoding(false); + const string ClefMediaType = "application/vnd.serilog.clef"; + + readonly ActiveLogBufferMap _logBufferMap; + readonly SeqForwarderOutputConfig _outputConfig; + readonly ServerResponseProxy _serverResponseProxy; + + readonly JsonSerializer _rawSerializer = JsonSerializer.Create( + new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); + + public IngestionController(ActiveLogBufferMap logBufferMap, SeqForwarderOutputConfig outputConfig, ServerResponseProxy serverResponseProxy) + { + _logBufferMap = logBufferMap; + _outputConfig = outputConfig; + _serverResponseProxy = serverResponseProxy; + } + + IPAddress ClientHostIP => Request.HttpContext.Connection.RemoteIpAddress!; + + [HttpGet, Route("api/events/describe")] + public IActionResult Resources() + { + return Content("{\"Links\":{\"Raw\":\"/api/events/raw{?clef}\"}}", "application/json", Encoding); + } + + [HttpPost, Route("api/events/raw")] + public async Task Ingest() + { + var clef = DefaultedBoolQuery("clef"); + + if (clef) + return await IngestCompactFormat(); + + var contentType = (string?) Request.Headers[HeaderNames.ContentType]; + if (contentType != null && contentType.StartsWith(ClefMediaType)) + return await IngestCompactFormat(); + + return IngestRawFormat(); + } + + IActionResult IngestRawFormat() + { + // The compact format ingestion path works with async IO. + HttpContext.Features.Get()!.AllowSynchronousIO = true; + + JObject posted; + try + { + posted = _rawSerializer.Deserialize(new JsonTextReader(new StreamReader(Request.Body))) ?? + throw new RequestProcessingException("Request body payload is JSON `null`."); + } + catch (Exception ex) + { + IngestionLog.ForClient(ClientHostIP).Debug(ex,"Rejecting payload due to invalid JSON, request body could not be parsed"); + throw new RequestProcessingException("Invalid raw event JSON, body could not be parsed."); + } + + if (!(posted.TryGetValue("events", StringComparison.Ordinal, out var eventsToken) || + posted.TryGetValue("Events", StringComparison.Ordinal, out eventsToken))) + { + IngestionLog.ForClient(ClientHostIP).Debug("Rejecting payload due to invalid JSON structure"); + throw new RequestProcessingException("Invalid raw event JSON, body must contain an 'Events' array."); + } + + if (!(eventsToken is JArray events)) + { + IngestionLog.ForClient(ClientHostIP).Debug("Rejecting payload due to invalid Events property structure"); + throw new RequestProcessingException("Invalid raw event JSON, the 'Events' property must be an array."); + } + + var encoded = EncodeRawEvents(events); + return Enqueue(encoded); + } + + async Task IngestCompactFormat() + { + var rawFormat = new List(); + var reader = new StreamReader(Request.Body); + + var line = await reader.ReadLineAsync(); + var lineNumber = 1; + + while (line != null) + { + if (!string.IsNullOrWhiteSpace(line)) + { + JObject item; + try + { + item = _rawSerializer.Deserialize(new JsonTextReader(new StringReader(line))) ?? + throw new RequestProcessingException("Request body payload is JSON `null`."); + } + catch (Exception ex) + { + IngestionLog.ForPayload(ClientHostIP, line).Debug(ex, "Rejecting CLEF payload due to invalid JSON, item could not be parsed"); + throw new RequestProcessingException($"Invalid raw event JSON, item on line {lineNumber} could not be parsed."); + } + + if (!EventSchema.FromClefFormat(lineNumber, item, out var evt, out var err)) + { + IngestionLog.ForPayload(ClientHostIP, line).Debug("Rejecting CLEF payload due to invalid event JSON structure: {NormalizationError}", err); + throw new RequestProcessingException(err); + } + + rawFormat.Add(evt); + } + + line = await reader.ReadLineAsync(); + ++lineNumber; + } + + var encoded = EncodeRawEvents(rawFormat); + return Enqueue(encoded); + } + + byte[][] EncodeRawEvents(ICollection events) + { + var encoded = new byte[events.Count][]; + var i = 0; + foreach (var e in events) + { + var s = e.ToString(Formatting.None); + var payload = Encoding.UTF8.GetBytes(s); + + if (payload.Length > (int) _outputConfig.EventBodyLimitBytes) + { + IngestionLog.ForPayload(ClientHostIP, s).Debug("An oversized event was dropped"); + + var jo = e as JObject; + // ReSharper disable SuspiciousTypeConversion.Global + var timestamp = (string?) (dynamic?) jo?.GetValue("Timestamp") ?? DateTime.UtcNow.ToString("o"); + var level = (string?) (dynamic?) jo?.GetValue("Level") ?? "Warning"; + + if (jo != null) + { + jo.Remove("Timestamp"); + jo.Remove("Level"); + } + + var startToLog = (int) Math.Min(_outputConfig.EventBodyLimitBytes / 2, 1024); + var compactPrefix = e.ToString(Formatting.None).Substring(0, startToLog); + + encoded[i] = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new + { + Timestamp = timestamp, + MessageTemplate = "Seq Forwarder received and dropped an oversized event", + Level = level, + Properties = new + { + Partial = compactPrefix, + Environment.MachineName, + _outputConfig.EventBodyLimitBytes, + PayloadBytes = payload.Length + } + })); + } + else + { + encoded[i] = payload; + } + + i++; + } + + return encoded; + } + + IActionResult Enqueue(byte[][] encodedEvents) + { + var apiKey = GetRequestApiKeyToken(); + _logBufferMap.GetLogBuffer(apiKey).Enqueue(encodedEvents); + + var response = Content(_serverResponseProxy.GetResponseText(apiKey), "application/json", Encoding); + response.StatusCode = (int)HttpStatusCode.Created; + return response; + } + + string? GetRequestApiKeyToken() + { + var apiKeyToken = Request.Headers[SeqApi.ApiKeyHeaderName].FirstOrDefault(); + + if (string.IsNullOrWhiteSpace(apiKeyToken)) + apiKeyToken = Request.Query["apiKey"]; + + var normalized = apiKeyToken?.Trim(); + if (string.IsNullOrEmpty(normalized)) + return null; + + return normalized; + } + + bool DefaultedBoolQuery(string queryParameterName) + { + var parameter = Request.Query[queryParameterName]; + if (parameter.Count != 1) + return false; + + var value = (string?) parameter; + + if (value == "" && ( + Request.QueryString.Value!.Contains($"&{queryParameterName}=") || + Request.QueryString.Value.Contains($"?{queryParameterName}="))) + { + return false; + } + + return "true".Equals(value, StringComparison.OrdinalIgnoreCase) || value == "" || value == queryParameterName; + } + } +} diff --git a/src/SeqCli/Forwarder/Web/Host/ServerService.cs b/src/SeqCli/Forwarder/Web/Host/ServerService.cs new file mode 100644 index 00000000..98d0d5dc --- /dev/null +++ b/src/SeqCli/Forwarder/Web/Host/ServerService.cs @@ -0,0 +1,67 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using Microsoft.Extensions.Hosting; +using Seq.Forwarder.Diagnostics; +using Seq.Forwarder.Multiplexing; +using Serilog; + +namespace Seq.Forwarder.Web.Host +{ + class ServerService + { + readonly ActiveLogBufferMap _logBufferMap; + readonly IHost _host; + readonly string _listenUri; + + public ServerService(ActiveLogBufferMap logBufferMap, IHost host, string listenUri) + { + _logBufferMap = logBufferMap; + _host = host; + _listenUri = listenUri; + } + + public void Start() + { + try + { + Log.Debug("Starting HTTP server..."); + + _host.Start(); + + Log.Information("Seq Forwarder listening on {ListenUri}", _listenUri); + IngestionLog.Log.Debug("Seq Forwarder is accepting events"); + + _logBufferMap.Load(); + _logBufferMap.Start(); + } + catch (Exception ex) + { + Log.Fatal(ex, "Error running the server application"); + throw; + } + } + + public void Stop() + { + Log.Debug("Seq Forwarder stopping"); + + _host.StopAsync().Wait(); + _logBufferMap.Stop(); + + Log.Information("Seq Forwarder stopped cleanly"); + } + } +} diff --git a/src/SeqCli/Forwarder/Web/Host/Startup.cs b/src/SeqCli/Forwarder/Web/Host/Startup.cs new file mode 100644 index 00000000..57379a33 --- /dev/null +++ b/src/SeqCli/Forwarder/Web/Host/Startup.cs @@ -0,0 +1,40 @@ +using System.Text; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; + +namespace Seq.Forwarder.Web.Host +{ + class Startup + { + public void ConfigureServices(IServiceCollection serviceCollection) + { + serviceCollection.AddMvc(); + } + + public void Configure(IApplicationBuilder app) + { + app.Use(async (context, next) => + { + try + { + await next(); + } + catch (RequestProcessingException rex) + { + if (context.Response.HasStarted) + throw; + + context.Response.StatusCode = (int)rex.StatusCode; + context.Response.ContentType = "text/plain; charset=UTF-8"; + await context.Response.Body.WriteAsync(Encoding.UTF8.GetBytes(rex.Message)); + await context.Response.CompleteAsync(); + } + }); + app.UseRouting(); + app.UseEndpoints(endpoints => + { + endpoints.MapControllers(); + }); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/RequestProcessingException.cs b/src/SeqCli/Forwarder/Web/RequestProcessingException.cs new file mode 100644 index 00000000..d5968643 --- /dev/null +++ b/src/SeqCli/Forwarder/Web/RequestProcessingException.cs @@ -0,0 +1,30 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Net; + +namespace Seq.Forwarder.Web +{ + class RequestProcessingException : Exception + { + public RequestProcessingException(string message, HttpStatusCode statusCode = HttpStatusCode.BadRequest) + : base(message) + { + StatusCode = statusCode; + } + + public HttpStatusCode StatusCode { get; } + } +} diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index 54e9377f..5d83f459 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -1,7 +1,7 @@ - + Exe - net8.0;net8.0-windows + net8.0 seqcli ..\..\asset\SeqCli.ico win-x64;linux-x64;linux-musl-x64;osx-x64;linux-arm64;linux-musl-arm64;osx-arm64 @@ -12,21 +12,10 @@ seqcli default enable - true - true - true + false + false + false - - - WINDOWS - - - OSX - - - LINUX - - @@ -47,7 +36,9 @@ + + diff --git a/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs new file mode 100644 index 00000000..46188948 --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs @@ -0,0 +1,83 @@ +using System.IO; +using System.Linq; +using Seq.Forwarder.Config; +using Seq.Forwarder.Cryptography; +using Seq.Forwarder.Multiplexing; +using Seq.Forwarder.Tests.Support; +using SeqCli.Tests.Support; +using Xunit; + +namespace Seq.Forwarder.Tests.Multiplexing +{ + public class ActiveLogBufferMapTests + { + [Fact] + public void AnEmptyMapCreatesNoFiles() + { + using var tmp = new TempFolder("Buffer"); + using var map = CreateActiveLogBufferMap(tmp); + Assert.Empty(Directory.GetFileSystemEntries(tmp.Path)); + } + + [Fact] + public void TheDefaultBufferWritesDataInTheBufferRoot() + { + using var tmp = new TempFolder("Buffer"); + using var map = CreateActiveLogBufferMap(tmp); + var entry = map.GetLogBuffer(null); + Assert.NotNull(entry); + Assert.True(File.Exists(Path.Combine(tmp.Path, "data.mdb"))); + Assert.Empty(Directory.GetDirectories(tmp.Path)); + Assert.Same(entry, map.GetLogBuffer(null)); + } + + [Fact] + public void ApiKeySpecificBuffersWriteDataToSubfolders() + { + using var tmp = new TempFolder("Buffer"); + using var map = CreateActiveLogBufferMap(tmp); + string key1 = Some.ApiKey(), key2 = Some.ApiKey(); + var entry1 = map.GetLogBuffer(key1); + var entry2 = map.GetLogBuffer(key2); + + Assert.NotNull(entry1); + Assert.NotNull(entry2); + Assert.Same(entry1, map.GetLogBuffer(key1)); + Assert.NotSame(entry1, entry2); + var subdirs = Directory.GetDirectories(tmp.Path); + Assert.Equal(2, subdirs.Length); + Assert.True(File.Exists(Path.Combine(subdirs[0], "data.mdb"))); + Assert.True(File.Exists(Path.Combine(subdirs[0], ".apikey"))); + } + + [Fact] + public void EntriesSurviveReloads() + { + var apiKey = Some.ApiKey(); + var value = Some.Bytes(100); + + using var tmp = new TempFolder("Buffer"); + using (var map = CreateActiveLogBufferMap(tmp)) + { + map.GetLogBuffer(null).Enqueue(new[] {value}); + map.GetLogBuffer(apiKey).Enqueue(new[] {value}); + } + + using (var map = CreateActiveLogBufferMap(tmp)) + { + var first = map.GetLogBuffer(null).Peek(0).Single(); + var second = map.GetLogBuffer(apiKey).Peek(0).Single(); + Assert.Equal(value, first.Value); + Assert.Equal(value, second.Value); + } + } + + static ActiveLogBufferMap CreateActiveLogBufferMap(TempFolder tmp) + { + var config = new SeqForwarderConfig(); + var map = new ActiveLogBufferMap(tmp.Path, config.Storage, config.Output, new InertLogShipperFactory(), StringDataProtector.CreatePlatformDefault()); + map.Load(); + return map; + } + } +} diff --git a/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs b/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs new file mode 100644 index 00000000..2032215e --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs @@ -0,0 +1,73 @@ +using System.IO; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Seq.Forwarder.Schema; +using Xunit; + +namespace Seq.Forwarder.Tests.Schema +{ + public class EventSchemaTests + { + static readonly JsonSerializer RawSerializer = JsonSerializer.Create( + new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); + + [Fact] + public void ClefNormalizationAcceptsDuplicateRenderings() + { + var payload = "{\"@t\": \"2015-05-09T12:09:08.12345Z\"," + + " \"@mt\": \"{A:000} and {A:000}\"," + + " \"@r\": [\"424\",\"424\"]}"; + + AssertCanNormalizeClef(payload); + } + + [Fact] + public void ClefNormalizationPropagatesRenderings() + { + const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12,\"@r\":[\"012\"]}"; + var evt = AssertCanNormalizeClef(payload); + Assert.Single(evt.Renderings); + } + + [Fact] + public void ClefNormalizationIgnoresMissingRenderings() + { + const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12}"; + AssertCanNormalizeClef(payload); + } + + [Fact] + public void ClefNormalizationFixesTooFewRenderings1() + { + const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12,\"@r\":[]}"; + var evt = AssertCanNormalizeClef(payload); + Assert.Null(evt.Renderings); + } + + [Fact] + public void ClefNormalizationFixesTooFewRenderings2() + { + const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000} {Q:x}!\",\"P\":12,\"@r\":[\"012\"]}"; + var evt = AssertCanNormalizeClef(payload); + Assert.Null(evt.Renderings); + } + + [Fact] + public void ClefNormalizationIgnoresTooManyRenderings() + { + const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12,\"@r\":[\"012\",\"013\"]}"; + var evt = AssertCanNormalizeClef(payload); + Assert.Null(evt.Renderings); + } + + static dynamic AssertCanNormalizeClef(string payload) + { + var jo = RawSerializer.Deserialize(new JsonTextReader(new StringReader(payload)))!; + + var valid = EventSchema.FromClefFormat(1, jo, out var rawFormat, out var error); + Assert.True(valid, error); + Assert.NotNull(rawFormat); + return rawFormat!; + } + } +} \ No newline at end of file diff --git a/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs b/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs new file mode 100644 index 00000000..1ac2db7c --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs @@ -0,0 +1,49 @@ +using Seq.Forwarder.Multiplexing; +using Seq.Forwarder.Shipper; +using Seq.Forwarder.Tests.Support; +using SeqCli.Tests.Support; +using Xunit; + +namespace Seq.Forwarder.Tests.Shipper +{ + public class ServerResponseProxyTests + { + [Fact] + public void WhenNoResponseRecordedEmptyIsReturned() + { + var proxy = new ServerResponseProxy(); + var response = proxy.GetResponseText(Some.ApiKey()); + Assert.Equal("{}", response); + } + + [Fact] + public void WhenApiKeysDontMatchEmptyResponseReturned() + { + var proxy = new ServerResponseProxy(); + proxy.SuccessResponseReturned(Some.ApiKey(), "this is never used"); + var response = proxy.GetResponseText(Some.ApiKey()); + Assert.Equal("{}", response); + } + + [Fact] + public void WhenApiKeysMatchTheResponseIsReturned() + { + var proxy = new ServerResponseProxy(); + var apiKey = Some.ApiKey(); + var responseText = "some response"; + proxy.SuccessResponseReturned(apiKey, responseText); + var response = proxy.GetResponseText(apiKey); + Assert.Equal(responseText, response); + } + + [Fact] + public void NullApiKeysAreConsideredMatching() + { + var proxy = new ServerResponseProxy(); + var responseText = "some response"; + proxy.SuccessResponseReturned(null, responseText); + var response = proxy.GetResponseText(null); + Assert.Equal(responseText, response); + } + } +} diff --git a/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs b/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs new file mode 100644 index 00000000..e6468d00 --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs @@ -0,0 +1,151 @@ +using System.Collections.Generic; +using Seq.Forwarder.Storage; +using Seq.Forwarder.Tests.Support; +using SeqCli.Tests.Support; +using Xunit; + +namespace Seq.Forwarder.Tests.Storage +{ + public class LogBufferTests + { + const ulong DefaultBufferSize = 10 * 1024 * 1024; + + [Fact] + public void ANewLogBufferIsEmpty() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + var contents = buffer.Peek((int)DefaultBufferSize); + Assert.Empty(contents); + } + + [Fact] + public void PeekingDoesNotChangeState() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + buffer.Enqueue(new[] { Some.Bytes(140) }); + + var contents = buffer.Peek((int)DefaultBufferSize); + Assert.Single(contents); + + var remainder = buffer.Peek((int)DefaultBufferSize); + Assert.Single(remainder); + } + + [Fact] + public void EnqueuedEntriesAreDequeuedFifo() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue(new[] { a1, a2 }); + buffer.Enqueue(new[] { a3 }); + + var contents = buffer.Peek((int)DefaultBufferSize); + + Assert.Equal(3, contents.Length); + Assert.Equal(a1, contents[0].Value); + Assert.Equal(a2, contents[1].Value); + Assert.Equal(a3, contents[2].Value); + } + + [Fact] + public void EntriesOverLimitArePurgedFifo() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), 4096); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue(new[] { a1, a2, a3 }); + + var contents = buffer.Peek((int)DefaultBufferSize); + + Assert.Equal(2, contents.Length); + Assert.Equal(a2, contents[0].Value); + Assert.Equal(a3, contents[1].Value); + } + + [Fact] + public void SizeHintLimitsDequeuedEventCount() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue(new[] { a1, a2, a3 }); + + var contents = buffer.Peek(300); + + Assert.Equal(2, contents.Length); + Assert.Equal(a1, contents[0].Value); + Assert.Equal(a2, contents[1].Value); + } + + [Fact] + public void AtLeastOneEventIsAlwaysDequeued() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue(new[] { a1, a2, a3 }); + + var contents = buffer.Peek(30); + + Assert.Single(contents); + Assert.Equal(a1, contents[0].Value); + } + + [Fact] + public void GivingTheLastSeenEventKeyRemovesPrecedingEvents() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue(new[] { a1, a2, a3 }); + + var contents = buffer.Peek(420); + Assert.Equal(3, contents.Length); + + buffer.Dequeue(contents[2].Key); + + var remaining = buffer.Peek(420); + Assert.Empty(remaining); + } + + [Fact] + public void GivingTheLastSeeEventKeyLeavesSuccessiveEvents() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue(new[] { a1, a2, a3 }); + + var contents = buffer.Peek(30); + Assert.Single(contents); + + buffer.Enqueue(new [] { Some.Bytes(140) }); + + buffer.Dequeue(contents[0].Key); + + var remaining = buffer.Peek(420); + Assert.Equal(3, remaining.Length); + } + + [Fact] + public void EnumerationIsInOrder() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue(new[] { a1, a2, a3 }); + + var contents = new List(); + buffer.Enumerate((k, v) => + { + contents.Add(v); + }); + + Assert.Equal(3, contents.Count); + Assert.Equal(new[] { a1, a2, a3 }, contents); + } + } +} diff --git a/test/SeqCli.Tests/SeqCli.Tests.csproj b/test/SeqCli.Tests/SeqCli.Tests.csproj index 8383a9bc..30d682cc 100644 --- a/test/SeqCli.Tests/SeqCli.Tests.csproj +++ b/test/SeqCli.Tests/SeqCli.Tests.csproj @@ -15,6 +15,7 @@ + diff --git a/test/SeqCli.Tests/Support/Some.cs b/test/SeqCli.Tests/Support/Some.cs index 17d1f829..4d6f5e3b 100644 --- a/test/SeqCli.Tests/Support/Some.cs +++ b/test/SeqCli.Tests/Support/Some.cs @@ -1,12 +1,17 @@ using System; using System.Linq; +using System.Security.Cryptography; using Serilog.Events; using Serilog.Parsing; namespace SeqCli.Tests.Support; +#nullable enable + static class Some { + static readonly RandomNumberGenerator Rng = RandomNumberGenerator.Create(); + public static LogEvent LogEvent() { return new LogEvent( @@ -26,4 +31,16 @@ public static string UriString() { return "http://example.com"; } + + public static byte[] Bytes(int count) + { + var bytes = new byte[count]; + Rng.GetBytes(bytes); + return bytes; + } + + public static string ApiKey() + { + return string.Join("", Bytes(8).Select(v => v.ToString("x2")).ToArray()); + } } \ No newline at end of file diff --git a/test/SeqCli.Tests/Support/TempFolder.cs b/test/SeqCli.Tests/Support/TempFolder.cs new file mode 100644 index 00000000..f7d358ef --- /dev/null +++ b/test/SeqCli.Tests/Support/TempFolder.cs @@ -0,0 +1,51 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Runtime.CompilerServices; + +#nullable enable + +namespace Seq.Forwarder.Tests.Support +{ + class TempFolder : IDisposable + { + static readonly Guid Session = Guid.NewGuid(); + + public TempFolder(string name) + { + Path = System.IO.Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), + "Seq.Forwarder.Tests", + Session.ToString("n"), + name); + + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public void Dispose() + { + try + { + if (Directory.Exists(Path)) + Directory.Delete(Path, true); + } + catch (Exception ex) + { + Debug.WriteLine(ex); + } + } + + public static TempFolder ForCaller([CallerMemberName] string? caller = null) + { + if (caller == null) throw new ArgumentNullException(nameof(caller)); + return new TempFolder(caller); + } + + public string AllocateFilename(string? ext = null) + { + return System.IO.Path.Combine(Path, Guid.NewGuid().ToString("n") + "." + (ext ?? "tmp")); + } + } +} From 2f365923603be27b27ad81835b98dedc3298d94b Mon Sep 17 00:00:00 2001 From: KodrAus Date: Thu, 29 Feb 2024 14:43:23 +1000 Subject: [PATCH 10/51] stub out encryption config --- src/SeqCli/Config/SeqCliConfig.cs | 1 + .../Config/SeqCliEncryptionProviderConfig.cs | 24 ++++ src/SeqCli/Encryptor/ExternalEncryption.cs | 116 ++++++++++++++++++ src/SeqCli/Encryptor/IEncryption.cs | 7 ++ src/SeqCli/Encryptor/PlaintextEncryption.cs | 14 +++ .../Encryptor/WindowsNativeEncryption.cs | 31 +++++ src/SeqCli/SeqCli.csproj | 1 + src/SeqCli/Util/PasswordHash.cs | 28 +++++ 8 files changed, 222 insertions(+) create mode 100644 src/SeqCli/Config/SeqCliEncryptionProviderConfig.cs create mode 100644 src/SeqCli/Encryptor/ExternalEncryption.cs create mode 100644 src/SeqCli/Encryptor/IEncryption.cs create mode 100644 src/SeqCli/Encryptor/PlaintextEncryption.cs create mode 100644 src/SeqCli/Encryptor/WindowsNativeEncryption.cs create mode 100644 src/SeqCli/Util/PasswordHash.cs diff --git a/src/SeqCli/Config/SeqCliConfig.cs b/src/SeqCli/Config/SeqCliConfig.cs index 307f14b8..8ff11a08 100644 --- a/src/SeqCli/Config/SeqCliConfig.cs +++ b/src/SeqCli/Config/SeqCliConfig.cs @@ -54,6 +54,7 @@ public static void Write(SeqCliConfig data) public ConnectionConfig Connection { get; set; } = new ConnectionConfig(); public OutputConfig Output { get; set; } = new(); public ForwarderConfig Forwarder { get; set; } = new(); + public SeqCliEncryptionProviderConfig EncryptionProviderProvider { get; set; } = new SeqCliEncryptionProviderConfig(); public Dictionary Profiles { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); diff --git a/src/SeqCli/Config/SeqCliEncryptionProviderConfig.cs b/src/SeqCli/Config/SeqCliEncryptionProviderConfig.cs new file mode 100644 index 00000000..c7818e18 --- /dev/null +++ b/src/SeqCli/Config/SeqCliEncryptionProviderConfig.cs @@ -0,0 +1,24 @@ +// Copyright 2024 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace SeqCli.Config; + +public class SeqCliEncryptionProviderConfig +{ + public string? Encryptor { get; set; } + public string? EncryptorArgs { get; set; } + + public string? Decryptor { get; set; } + public string? DecryptorArgs { get; set; } +} \ No newline at end of file diff --git a/src/SeqCli/Encryptor/ExternalEncryption.cs b/src/SeqCli/Encryptor/ExternalEncryption.cs new file mode 100644 index 00000000..b9db753f --- /dev/null +++ b/src/SeqCli/Encryptor/ExternalEncryption.cs @@ -0,0 +1,116 @@ +using System; +using System.Buffers; +using System.Diagnostics; +using System.Text; +using System.Threading; +using SeqCli.Config; + +namespace SeqCli.Encryptor; + +public class ExternalEncryption : IEncryption +{ + public ExternalEncryption(SeqCliEncryptionProviderConfig providerConfig) + { + _encryptor = providerConfig.Encryptor!; + _encryptorArgs = providerConfig.EncryptorArgs; + + _decryptor = providerConfig.Decryptor!; + _decryptorArgs = providerConfig.DecryptorArgs; + } + + readonly string _encryptor; + readonly string? _encryptorArgs; + readonly string _decryptor; + readonly string? _decryptorArgs; + + public byte[] Encrypt(byte[] unencrypted) + { + var exit = Invoke(_encryptor, _encryptorArgs, unencrypted, out var encrypted, out var err); + if (exit != 0) + { + throw new Exception($"Encryptor failed with exit code {exit} and produced: {err}"); + } + + return encrypted; + } + + public byte[] Decrypt(byte[] encrypted) + { + var exit = Invoke(_decryptor, _decryptorArgs, encrypted, out var decrypted, out var err); + if (exit != 0) + { + throw new Exception($"Decryptor failed with exit code {exit} and produced: {err}"); + } + + return decrypted; + } + + static int Invoke(string fullExePath, string? args, byte[] stdin, out byte[] stdout, out string stderr) + { + var startInfo = new ProcessStartInfo + { + UseShellExecute = false, + RedirectStandardError = true, + RedirectStandardOutput = true, + WindowStyle = ProcessWindowStyle.Hidden, + CreateNoWindow = true, + ErrorDialog = false, + FileName = fullExePath, + Arguments = args + }; + + using var process = Process.Start(startInfo); + using var errorComplete = new ManualResetEvent(false); + + if (process == null) + throw new InvalidOperationException("The process did not start."); + + var stderrBuf = new StringBuilder(); + process.ErrorDataReceived += (o, e) => + { + if (e.Data == null) + // ReSharper disable once AccessToDisposedClosure + errorComplete.Set(); + else + stderrBuf.Append(e.Data); + }; + process.BeginErrorReadLine(); + + process.StandardInput.BaseStream.Write(stdin); + process.StandardInput.BaseStream.Close(); + + var stdoutBuf = ArrayPool.Shared.Rent(512); + var stdoutBufLength = 0; + while (true) + { + var remaining = stdoutBuf.Length - stdoutBufLength; + if (remaining == 0) + { + var newBuffer = ArrayPool.Shared.Rent(stdoutBuf.Length * 2); + stdoutBuf.CopyTo(newBuffer.AsSpan()); + + ArrayPool.Shared.Return(stdoutBuf); + stdoutBuf = newBuffer; + + remaining = stdoutBuf.Length - stdoutBufLength; + } + + var read = process.StandardOutput.BaseStream.Read(stdoutBuf, stdoutBufLength, remaining); + + if (read == 0) + { + break; + } + + stdoutBufLength += read; + } + + errorComplete.WaitOne(); + stderr = stderrBuf.ToString(); + + stdout = stdoutBuf.AsSpan()[..stdoutBufLength].ToArray(); + ArrayPool.Shared.Return(stdoutBuf); + + return process.ExitCode; + } +} \ No newline at end of file diff --git a/src/SeqCli/Encryptor/IEncryption.cs b/src/SeqCli/Encryptor/IEncryption.cs new file mode 100644 index 00000000..0294fa82 --- /dev/null +++ b/src/SeqCli/Encryptor/IEncryption.cs @@ -0,0 +1,7 @@ +namespace SeqCli.Encryptor; + +public interface IEncryption +{ + public byte[] Encrypt(byte[] unencrypted); + public byte[] Decrypt(byte[] encrypted); +} \ No newline at end of file diff --git a/src/SeqCli/Encryptor/PlaintextEncryption.cs b/src/SeqCli/Encryptor/PlaintextEncryption.cs new file mode 100644 index 00000000..53a8df3e --- /dev/null +++ b/src/SeqCli/Encryptor/PlaintextEncryption.cs @@ -0,0 +1,14 @@ +namespace SeqCli.Encryptor; + +class PlaintextEncryption : IEncryption +{ + public byte[] Encrypt(byte[] unencrypted) + { + return unencrypted; + } + + public byte[] Decrypt(byte[] encrypted) + { + return encrypted; + } +} \ No newline at end of file diff --git a/src/SeqCli/Encryptor/WindowsNativeEncryption.cs b/src/SeqCli/Encryptor/WindowsNativeEncryption.cs new file mode 100644 index 00000000..323a82df --- /dev/null +++ b/src/SeqCli/Encryptor/WindowsNativeEncryption.cs @@ -0,0 +1,31 @@ +using System; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using SeqCli.Util; + +namespace SeqCli.Encryptor; + +public class WindowsNativeEncryption : IEncryption +{ + public byte[] Encrypt(byte[] unencrypted) + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + throw new PlatformNotSupportedException("Windows native encryption is only supported on Windows"); + + var salt = PasswordHash.GenerateSalt(); + var data = ProtectedData.Protect(unencrypted, salt, DataProtectionScope.LocalMachine); + + return [..data, ..salt]; + } + + public byte[] Decrypt(byte[] encrypted) + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + throw new PlatformNotSupportedException("Windows native encryption is only supported on Windows"); + + var data = encrypted[..^16]; + var salt = encrypted[^16..]; + + return ProtectedData.Unprotect(data, salt, DataProtectionScope.LocalMachine); + } +} \ No newline at end of file diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index 5d83f459..c29883b2 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -47,6 +47,7 @@ + diff --git a/src/SeqCli/Util/PasswordHash.cs b/src/SeqCli/Util/PasswordHash.cs new file mode 100644 index 00000000..91576624 --- /dev/null +++ b/src/SeqCli/Util/PasswordHash.cs @@ -0,0 +1,28 @@ +using System; +using System.Security.Cryptography; + +namespace SeqCli.Util; + +static class PasswordHash +{ + const int SaltSize = 16, + HashSize = 64, + HashIter = 500_000; + + public static byte[] GenerateSalt() + { + var salt = new byte[SaltSize]; + using var cp = RandomNumberGenerator.Create(); + cp.GetBytes(salt); + return salt; + } + + public static byte[] Calculate(string password, byte[] salt) + { + if (password == null) throw new ArgumentNullException(nameof(password)); + if (salt == null) throw new ArgumentNullException(nameof(salt)); + + using var algorithm = new Rfc2898DeriveBytes(password, salt, HashIter, HashAlgorithmName.SHA512); + return algorithm.GetBytes(HashSize); + } +} \ No newline at end of file From 6877c1f88779b018639de2d5b826e5df1d71dfb4 Mon Sep 17 00:00:00 2001 From: KodrAus Date: Thu, 29 Feb 2024 15:05:53 +1000 Subject: [PATCH 11/51] get things building on windows --- .../Cli/Commands/Forwarder/InstallCommand.cs | 102 +++++++++--------- .../Cli/Commands/Forwarder/RestartCommand.cs | 32 +++--- .../Cli/Commands/Forwarder/RunCommand.cs | 3 + .../Cli/Commands/Forwarder/StartCommand.cs | 28 ++--- .../Cli/Commands/Forwarder/StatusCommand.cs | 18 ++-- .../Cli/Commands/Forwarder/StopCommand.cs | 28 ++--- .../Commands/Forwarder/UninstallCommand.cs | 16 +-- .../DpapiMachineScopeDataProtection.cs | 2 + .../SeqForwarderWindowsService.cs | 2 + .../Forwarder/Util/ServiceConfiguration.cs | 20 ++-- src/SeqCli/Program.cs | 4 + src/SeqCli/SeqCli.csproj | 14 ++- test/SeqCli.EndToEnd/SeqCli.EndToEnd.csproj | 2 +- test/SeqCli.Tests/SeqCli.Tests.csproj | 2 +- 14 files changed, 160 insertions(+), 113 deletions(-) diff --git a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs index 7dbc9873..4fc70ed0 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs @@ -15,20 +15,26 @@ #if WINDOWS using System; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Runtime.InteropServices; using System.Security.AccessControl; using System.ServiceProcess; +using System.Threading.Tasks; using Seq.Forwarder.Cli.Features; using Seq.Forwarder.Config; using Seq.Forwarder.ServiceProcess; using Seq.Forwarder.Util; +using SeqCli; +using SeqCli.Cli; +using SeqCli.Cli.Features; // ReSharper disable once ClassNeverInstantiated.Global namespace Seq.Forwarder.Cli.Commands { [Command("forwarder", "install", "Install the Seq Forwarder as a Windows service")] + [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class InstallCommand : Command { readonly StoragePathFeature _storagePath; @@ -51,70 +57,70 @@ public InstallCommand() string ServiceUsername => _serviceCredentials.IsUsernameSpecified ? _serviceCredentials.Username : "NT AUTHORITY\\LocalService"; - protected override int Run(TextWriter cout) + protected override Task Run() { try { if (!_setup) { - Install(cout); - return 0; + Install(); + return Task.FromResult(0); } - var exit = Setup(cout); + var exit = Setup(); if (exit == 0) { Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine("Setup completed successfully."); Console.ResetColor(); } - return exit; + return Task.FromResult(exit); } catch (DirectoryNotFoundException dex) { - cout.WriteLine("Could not install the service, directory not found: " + dex.Message); - return -1; + Console.WriteLine("Could not install the service, directory not found: " + dex.Message); + return Task.FromResult(-1); } catch (Exception ex) { - cout.WriteLine("Could not install the service: " + ex.Message); - return -1; + Console.WriteLine("Could not install the service: " + ex.Message); + return Task.FromResult(-1); } } - int Setup(TextWriter cout) + int Setup() { ServiceController controller; try { - cout.WriteLine("Checking the status of the Seq Forwarder service..."); + Console.WriteLine("Checking the status of the Seq Forwarder service..."); controller = new ServiceController(SeqForwarderWindowsService.WindowsServiceName); - cout.WriteLine("Status is {0}", controller.Status); + Console.WriteLine("Status is {0}", controller.Status); } catch (InvalidOperationException) { - Install(cout); + Install(); var controller2 = new ServiceController(SeqForwarderWindowsService.WindowsServiceName); - return Start(controller2, cout); + return Start(controller2); } - cout.WriteLine("Service is installed; checking path and dependency configuration..."); - Reconfigure(controller, cout); + Console.WriteLine("Service is installed; checking path and dependency configuration..."); + Reconfigure(controller); if (controller.Status != ServiceControllerStatus.Running) - return Start(controller, cout); + return Start(controller); return 0; } - static void Reconfigure(ServiceController controller, TextWriter cout) + static void Reconfigure(ServiceController controller) { var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); - if (0 != CaptiveProcess.Run(sc, "config \"" + controller.ServiceName + "\" depend= Winmgmt/Tcpip/CryptSvc", cout.WriteLine, cout.WriteLine)) - cout.WriteLine("Could not reconfigure service dependencies; ignoring."); + if (0 != CaptiveProcess.Run(sc, "config \"" + controller.ServiceName + "\" depend= Winmgmt/Tcpip/CryptSvc", Console.WriteLine, Console.WriteLine)) + Console.WriteLine("Could not reconfigure service dependencies; ignoring."); - if (!ServiceConfiguration.GetServiceBinaryPath(controller, cout, out var path)) + if (!ServiceConfiguration.GetServiceBinaryPath(controller, out var path)) return; var current = "\"" + Path.Combine(AppDomain.CurrentDomain.BaseDirectory!, Program.BinaryName) + "\""; @@ -124,57 +130,57 @@ static void Reconfigure(ServiceController controller, TextWriter cout) var seqRun = path.IndexOf(Program.BinaryName + "\" run", StringComparison.OrdinalIgnoreCase); if (seqRun == -1) { - cout.WriteLine("Current binary path is an unrecognized format."); + Console.WriteLine("Current binary path is an unrecognized format."); return; } - cout.WriteLine("Existing service binary path is: {0}", path); + Console.WriteLine("Existing service binary path is: {0}", path); var trimmed = path.Substring((seqRun + Program.BinaryName + " ").Length); var newPath = current + trimmed; - cout.WriteLine("Updating service binary path configuration to: {0}", newPath); + Console.WriteLine("Updating service binary path configuration to: {0}", newPath); var escaped = newPath.Replace("\"", "\\\""); - if (0 != CaptiveProcess.Run(sc, "config \"" + controller.ServiceName + "\" binPath= \"" + escaped + "\"", cout.WriteLine, cout.WriteLine)) + if (0 != CaptiveProcess.Run(sc, "config \"" + controller.ServiceName + "\" binPath= \"" + escaped + "\"", Console.WriteLine, Console.WriteLine)) { - cout.WriteLine("Could not reconfigure service path; ignoring."); + Console.WriteLine("Could not reconfigure service path; ignoring."); return; } - cout.WriteLine("Service binary path reconfigured successfully."); + Console.WriteLine("Service binary path reconfigured successfully."); } - static int Start(ServiceController controller, TextWriter cout) + static int Start(ServiceController controller) { controller.Start(); if (controller.Status != ServiceControllerStatus.Running) { - cout.WriteLine("Waiting up to 60 seconds for the service to start (currently: " + controller.Status + ")..."); + Console.WriteLine("Waiting up to 60 seconds for the service to start (currently: " + controller.Status + ")..."); controller.WaitForStatus(ServiceControllerStatus.Running, TimeSpan.FromSeconds(60)); } if (controller.Status == ServiceControllerStatus.Running) { - cout.WriteLine("Started."); + Console.WriteLine("Started."); return 0; } - cout.WriteLine("The service hasn't started successfully."); + Console.WriteLine("The service hasn't started successfully."); return -1; } [DllImport("shlwapi.dll")] static extern bool PathIsNetworkPath(string pszPath); - void Install(TextWriter cout) + void Install() { - cout.WriteLine("Installing service..."); + Console.WriteLine("Installing service..."); if (PathIsNetworkPath(_storagePath.StorageRootPath)) throw new ArgumentException("Seq requires a local (or SAN) storage location; network shares are not supported."); - cout.WriteLine($"Updating the configuration in {_storagePath.ConfigFilePath}..."); + Console.WriteLine($"Updating the configuration in {_storagePath.ConfigFilePath}..."); var config = SeqForwarderConfig.ReadOrInit(_storagePath.ConfigFilePath); if (!string.IsNullOrEmpty(_listenUri.ListenUri)) @@ -190,21 +196,21 @@ void Install(TextWriter cout) "If a service user account is specified, a password for the account must also be specified."); // https://technet.microsoft.com/en-us/library/cc794944(v=ws.10).aspx - cout.WriteLine($"Ensuring {_serviceCredentials.Username} is granted 'Log on as a Service' rights..."); + Console.WriteLine($"Ensuring {_serviceCredentials.Username} is granted 'Log on as a Service' rights..."); AccountRightsHelper.EnsureServiceLogOnRights(_serviceCredentials.Username); } - cout.WriteLine($"Granting {ServiceUsername} rights to {_storagePath.StorageRootPath}..."); + Console.WriteLine($"Granting {ServiceUsername} rights to {_storagePath.StorageRootPath}..."); GiveFullControl(_storagePath.StorageRootPath); - cout.WriteLine($"Granting {ServiceUsername} rights to {config.Diagnostics.InternalLogPath}..."); + Console.WriteLine($"Granting {ServiceUsername} rights to {config.Diagnostics.InternalLogPath}..."); GiveFullControl(config.Diagnostics.InternalLogPath); var listenUri = MakeListenUriReservationPattern(config.Api.ListenUri); - cout.WriteLine($"Adding URL reservation at {listenUri} for {ServiceUsername}..."); - var netshResult = CaptiveProcess.Run("netsh", $"http add urlacl url={listenUri} user=\"{ServiceUsername}\"", cout.WriteLine, cout.WriteLine); + Console.WriteLine($"Adding URL reservation at {listenUri} for {ServiceUsername}..."); + var netshResult = CaptiveProcess.Run("netsh", $"http add urlacl url={listenUri} user=\"{ServiceUsername}\"", Console.WriteLine, Console.WriteLine); if (netshResult != 0) - cout.WriteLine($"Could not add URL reservation for {listenUri}: `netsh` returned {netshResult}; ignoring"); + Console.WriteLine($"Could not add URL reservation for {listenUri}: `netsh` returned {netshResult}; ignoring"); var exePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory!, Program.BinaryName); var forwarderRunCmdline = $"\"{exePath}\" run --storage=\"{_storagePath.StorageRootPath}\""; @@ -220,19 +226,19 @@ void Install(TextWriter cout) scCmdline += $" obj= {_serviceCredentials.Username} password= {_serviceCredentials.Password}"; var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); - if (0 != CaptiveProcess.Run(sc, scCmdline, cout.WriteLine, cout.WriteLine)) + if (0 != CaptiveProcess.Run(sc, scCmdline, Console.WriteLine, Console.WriteLine)) { throw new ArgumentException("Service setup failed"); } - cout.WriteLine("Setting service restart policy..."); - if (0 != CaptiveProcess.Run(sc, $"failure \"{SeqForwarderWindowsService.WindowsServiceName}\" actions= restart/60000/restart/60000/restart/60000// reset= 600000", cout.WriteLine, cout.WriteLine)) - cout.WriteLine("Could not set service restart policy; ignoring"); - cout.WriteLine("Setting service description..."); - if (0 != CaptiveProcess.Run(sc, $"description \"{SeqForwarderWindowsService.WindowsServiceName}\" \"Durable storage and forwarding of application log events\"", cout.WriteLine, cout.WriteLine)) - cout.WriteLine("Could not set service description; ignoring"); + Console.WriteLine("Setting service restart policy..."); + if (0 != CaptiveProcess.Run(sc, $"failure \"{SeqForwarderWindowsService.WindowsServiceName}\" actions= restart/60000/restart/60000/restart/60000// reset= 600000", Console.WriteLine, Console.WriteLine)) + Console.WriteLine("Could not set service restart policy; ignoring"); + Console.WriteLine("Setting service description..."); + if (0 != CaptiveProcess.Run(sc, $"description \"{SeqForwarderWindowsService.WindowsServiceName}\" \"Durable storage and forwarding of application log events\"", Console.WriteLine, Console.WriteLine)) + Console.WriteLine("Could not set service description; ignoring"); - cout.WriteLine("Service installed successfully."); + Console.WriteLine("Service installed successfully."); } void GiveFullControl(string target) diff --git a/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs index fb97172a..bf04e666 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs @@ -15,18 +15,22 @@ #if WINDOWS using System; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.ServiceProcess; +using System.Threading.Tasks; using Seq.Forwarder.ServiceProcess; +using SeqCli.Cli; // ReSharper disable UnusedType.Global namespace Seq.Forwarder.Cli.Commands { [Command("forwarder", "restart", "Restart the Windows service")] + [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class RestartCommand : Command { - protected override int Run(TextWriter cout) + protected override Task Run() { try { @@ -34,47 +38,47 @@ protected override int Run(TextWriter cout) if (controller.Status != ServiceControllerStatus.Stopped) { - cout.WriteLine("Stopping {0}...", controller.ServiceName); + Console.WriteLine("Stopping {0}...", controller.ServiceName); controller.Stop(); if (controller.Status != ServiceControllerStatus.Stopped) { - cout.WriteLine("Waiting up to 60 seconds for the service to stop (currently: " + + Console.WriteLine("Waiting up to 60 seconds for the service to stop (currently: " + controller.Status + ")..."); controller.WaitForStatus(ServiceControllerStatus.Stopped, TimeSpan.FromSeconds(60)); } if (controller.Status != ServiceControllerStatus.Stopped) { - cout.WriteLine("The service hasn't stopped successfully."); - return -1; + Console.WriteLine("The service hasn't stopped successfully."); + return Task.FromResult(-1); } } - cout.WriteLine("Starting {0}...", controller.ServiceName); + Console.WriteLine("Starting {0}...", controller.ServiceName); controller.Start(); if (controller.Status != ServiceControllerStatus.Running) { - cout.WriteLine("Waiting up to 15 seconds for the service to start (currently: " + controller.Status + ")..."); + Console.WriteLine("Waiting up to 15 seconds for the service to start (currently: " + controller.Status + ")..."); controller.WaitForStatus(ServiceControllerStatus.Running, TimeSpan.FromSeconds(15)); } if (controller.Status == ServiceControllerStatus.Running) { - cout.WriteLine("Started."); - return 0; + Console.WriteLine("Started."); + return Task.FromResult(0); } - cout.WriteLine("The service hasn't started successfully."); - return -1; + Console.WriteLine("The service hasn't started successfully."); + return Task.FromResult(-1); } catch (Exception ex) { - cout.WriteLine(ex.Message); + Console.WriteLine(ex.Message); if (ex.InnerException != null) - cout.WriteLine(ex.InnerException.Message); - return 1; + Console.WriteLine(ex.InnerException.Message); + return Task.FromResult(1); } } } diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index 0af8e266..4c39c38b 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -18,8 +18,10 @@ using Serilog.Events; using Serilog.Formatting.Compact; using System; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net; +using System.Security.Cryptography.X509Certificates; using System.Threading.Tasks; using Autofac.Extensions.DependencyInjection; using Microsoft.AspNetCore.Hosting; @@ -198,6 +200,7 @@ static string GetRollingLogFilePathFormat(string internalLogPath) return Path.Combine(internalLogPath, "seq-forwarder-.log"); } + [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] static int RunService(ServerService service) { #if WINDOWS diff --git a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs index 2b48ca4f..150c9f6d 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs @@ -15,50 +15,54 @@ #if WINDOWS using System; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.ServiceProcess; +using System.Threading.Tasks; using Seq.Forwarder.ServiceProcess; +using SeqCli.Cli; namespace Seq.Forwarder.Cli.Commands { [Command("forwarder", "start", "Start the Windows service")] + [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class StartCommand : Command { - protected override int Run(TextWriter cout) + protected override Task Run() { try { var controller = new ServiceController(SeqForwarderWindowsService.WindowsServiceName); if (controller.Status != ServiceControllerStatus.Stopped) { - cout.WriteLine("Cannot start {0}, current status is: {1}", controller.ServiceName, controller.Status); - return -1; + Console.WriteLine("Cannot start {0}, current status is: {1}", controller.ServiceName, controller.Status); + return Task.FromResult(-1); } - cout.WriteLine("Starting {0}...", controller.ServiceName); + Console.WriteLine("Starting {0}...", controller.ServiceName); controller.Start(); if (controller.Status != ServiceControllerStatus.Running) { - cout.WriteLine("Waiting up to 15 seconds for the service to start (currently: " + controller.Status + ")..."); + Console.WriteLine("Waiting up to 15 seconds for the service to start (currently: " + controller.Status + ")..."); controller.WaitForStatus(ServiceControllerStatus.Running, TimeSpan.FromSeconds(15)); } if (controller.Status == ServiceControllerStatus.Running) { - cout.WriteLine("Started."); - return 0; + Console.WriteLine("Started."); + return Task.FromResult(0); } - cout.WriteLine("The service hasn't started successfully."); - return -1; + Console.WriteLine("The service hasn't started successfully."); + return Task.FromResult(-1); } catch (Exception ex) { - cout.WriteLine(ex.Message); + Console.WriteLine(ex.Message); if (ex.InnerException != null) - cout.WriteLine(ex.InnerException.Message); - return -1; + Console.WriteLine(ex.InnerException.Message); + return Task.FromResult(-1); } } } diff --git a/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs index c9261d93..40b05669 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs @@ -15,35 +15,39 @@ #if WINDOWS using System; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.ServiceProcess; +using System.Threading.Tasks; using Seq.Forwarder.ServiceProcess; +using SeqCli.Cli; namespace Seq.Forwarder.Cli.Commands { [Command("forwarder", "status", "Show the status of the Seq Forwarder service")] + [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class StatusCommand : Command { - protected override int Run(TextWriter cout) + protected override Task Run() { try { var controller = new ServiceController(SeqForwarderWindowsService.WindowsServiceName); - cout.WriteLine("The Seq Forwarder service is installed and {0}.", controller.Status.ToString().ToLowerInvariant()); + Console.WriteLine("The Seq Forwarder service is installed and {0}.", controller.Status.ToString().ToLowerInvariant()); } catch (InvalidOperationException) { - cout.WriteLine("The Seq Forwarder service is not installed."); + Console.WriteLine("The Seq Forwarder service is not installed."); } catch (Exception ex) { - cout.WriteLine(ex.Message); + Console.WriteLine(ex.Message); if (ex.InnerException != null) - cout.WriteLine(ex.InnerException.Message); - return 1; + Console.WriteLine(ex.InnerException.Message); + return Task.FromResult(1); } - return 0; + return Task.FromResult(1); } } } diff --git a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs index 66c16637..d5ee2f1a 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs @@ -15,16 +15,20 @@ #if WINDOWS using System; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.ServiceProcess; +using System.Threading.Tasks; using Seq.Forwarder.ServiceProcess; +using SeqCli.Cli; namespace Seq.Forwarder.Cli.Commands { [Command("forwarder", "stop", "Stop the Windows service")] + [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class StopCommand : Command { - protected override int Run(TextWriter cout) + protected override Task Run() { try { @@ -32,34 +36,34 @@ protected override int Run(TextWriter cout) if (controller.Status != ServiceControllerStatus.Running) { - cout.WriteLine("Cannot stop {0}, current status is: {1}", controller.ServiceName, controller.Status); - return -1; + Console.WriteLine("Cannot stop {0}, current status is: {1}", controller.ServiceName, controller.Status); + return Task.FromResult(-1); } - cout.WriteLine("Stopping {0}...", controller.ServiceName); + Console.WriteLine("Stopping {0}...", controller.ServiceName); controller.Stop(); if (controller.Status != ServiceControllerStatus.Stopped) { - cout.WriteLine("Waiting up to 60 seconds for the service to stop (currently: " + controller.Status + ")..."); + Console.WriteLine("Waiting up to 60 seconds for the service to stop (currently: " + controller.Status + ")..."); controller.WaitForStatus(ServiceControllerStatus.Stopped, TimeSpan.FromSeconds(60)); } if (controller.Status == ServiceControllerStatus.Stopped) { - cout.WriteLine("Stopped."); - return 0; + Console.WriteLine("Stopped."); + return Task.FromResult(0); } - cout.WriteLine("The service hasn't stopped successfully."); - return -1; + Console.WriteLine("The service hasn't stopped successfully."); + return Task.FromResult(-1); } catch (Exception ex) { - cout.WriteLine(ex.Message); + Console.WriteLine(ex.Message); if (ex.InnerException != null) - cout.WriteLine(ex.InnerException.Message); - return -1; + Console.WriteLine(ex.InnerException.Message); + return Task.FromResult(-1); } } } diff --git a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs index ee717ab5..96fbed59 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs @@ -16,32 +16,34 @@ using System; using System.IO; +using System.Threading.Tasks; using Seq.Forwarder.ServiceProcess; using Seq.Forwarder.Util; +using SeqCli.Cli; namespace Seq.Forwarder.Cli.Commands { [Command("forwarder", "uninstall", "Uninstall the Windows service")] class UninstallCommand : Command { - protected override int Run(TextWriter cout) + protected override Task Run() { try { - cout.WriteLine("Uninstalling service..."); + Console.WriteLine("Uninstalling service..."); var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); - var exitCode = CaptiveProcess.Run(sc, $"delete \"{SeqForwarderWindowsService.WindowsServiceName}\"", cout.WriteLine, cout.WriteLine); + var exitCode = CaptiveProcess.Run(sc, $"delete \"{SeqForwarderWindowsService.WindowsServiceName}\"", Console.WriteLine, Console.WriteLine); if (exitCode != 0) throw new InvalidOperationException($"The `sc.exe delete` call failed with exit code {exitCode}."); - cout.WriteLine("Service uninstalled successfully."); - return 0; + Console.WriteLine("Service uninstalled successfully."); + return Task.FromResult(0); } catch (Exception ex) { - cout.WriteLine("Could not uninstall the service: " + ex.Message); - return -1; + Console.WriteLine("Could not uninstall the service: " + ex.Message); + return Task.FromResult(-1); } } } diff --git a/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs b/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs index 134eaaa3..635bb415 100644 --- a/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs +++ b/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs @@ -15,11 +15,13 @@ #if WINDOWS using System; +using System.Diagnostics.CodeAnalysis; using System.Security.Cryptography; using System.Text; namespace Seq.Forwarder.Cryptography { + [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class DpapiMachineScopeDataProtect : IStringDataProtector { public string Unprotect(string @protected) diff --git a/src/SeqCli/Forwarder/ServiceProcess/SeqForwarderWindowsService.cs b/src/SeqCli/Forwarder/ServiceProcess/SeqForwarderWindowsService.cs index 013f67be..a77efec1 100644 --- a/src/SeqCli/Forwarder/ServiceProcess/SeqForwarderWindowsService.cs +++ b/src/SeqCli/Forwarder/ServiceProcess/SeqForwarderWindowsService.cs @@ -14,12 +14,14 @@ #if WINDOWS +using System.Diagnostics.CodeAnalysis; using System.Net; using System.ServiceProcess; using Seq.Forwarder.Web.Host; namespace Seq.Forwarder.ServiceProcess { + [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class SeqForwarderWindowsService : ServiceBase { readonly ServerService _serverService; diff --git a/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs b/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs index 2fda1fd5..b15c4775 100644 --- a/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs +++ b/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs @@ -23,16 +23,17 @@ namespace Seq.Forwarder.Util { + [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] public static class ServiceConfiguration { - public static bool GetServiceBinaryPath(ServiceController controller, TextWriter cout, [MaybeNullWhen(false)] out string path) + public static bool GetServiceBinaryPath(ServiceController controller, [MaybeNullWhen(false)] out string path) { var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); var config = new StringBuilder(); - if (0 != CaptiveProcess.Run(sc, "qc \"" + controller.ServiceName + "\"", l => config.AppendLine(l), cout.WriteLine)) + if (0 != CaptiveProcess.Run(sc, "qc \"" + controller.ServiceName + "\"", l => config.AppendLine(l), Console.WriteLine)) { - cout.WriteLine("Could not query service path; ignoring."); + Console.WriteLine("Could not query service path; ignoring."); path = null; return false; } @@ -46,7 +47,7 @@ public static bool GetServiceBinaryPath(ServiceController controller, TextWriter if (line == null) { - cout.WriteLine("No existing binary path could be determined."); + Console.WriteLine("No existing binary path could be determined."); path = null; return false; } @@ -55,17 +56,16 @@ public static bool GetServiceBinaryPath(ServiceController controller, TextWriter return true; } - static bool GetServiceCommandLine(string serviceName, TextWriter cout, [MaybeNullWhen(false)] out string path) + static bool GetServiceCommandLine(string serviceName, [MaybeNullWhen(false)] out string path) { if (serviceName == null) throw new ArgumentNullException(nameof(serviceName)); - if (cout == null) throw new ArgumentNullException(nameof(cout)); var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); var config = new StringBuilder(); - if (0 != CaptiveProcess.Run(sc, "qc \"" + serviceName + "\"", l => config.AppendLine(l), cout.WriteLine)) + if (0 != CaptiveProcess.Run(sc, "qc \"" + serviceName + "\"", l => config.AppendLine(l), Console.WriteLine)) { - cout.WriteLine("Could not query service path; ignoring."); + Console.WriteLine("Could not query service path; ignoring."); path = null; return false; } @@ -79,7 +79,7 @@ static bool GetServiceCommandLine(string serviceName, TextWriter cout, [MaybeNul if (line == null) { - cout.WriteLine("No existing binary path could be determined."); + Console.WriteLine("No existing binary path could be determined."); path = null; return false; } @@ -92,7 +92,7 @@ public static bool GetServiceStoragePath(string serviceName, out string? storage { if (serviceName == null) throw new ArgumentNullException(nameof(serviceName)); - if (GetServiceCommandLine(serviceName, new StringWriter(), out var binpath) && + if (GetServiceCommandLine(serviceName, out var binpath) && binpath.Contains("--storage=\"")) { var start = binpath.IndexOf("--storage=\"", StringComparison.Ordinal) + 11; diff --git a/src/SeqCli/Program.cs b/src/SeqCli/Program.cs index e7d55cdc..f23952a4 100644 --- a/src/SeqCli/Program.cs +++ b/src/SeqCli/Program.cs @@ -26,6 +26,10 @@ namespace SeqCli; class Program { +#if WINDOWS + public const string BinaryName = "seqcli.exe"; +#endif + static async Task Main(string[] args) { var levelSwitch = new LoggingLevelSwitch(LogEventLevel.Error); diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index c29883b2..fbb39cf1 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -15,6 +15,18 @@ false false false + true + true + true + + + WINDOWS + + + OSX + + + LINUX @@ -43,11 +55,11 @@ + - diff --git a/test/SeqCli.EndToEnd/SeqCli.EndToEnd.csproj b/test/SeqCli.EndToEnd/SeqCli.EndToEnd.csproj index 2c417644..c5ee3ea4 100644 --- a/test/SeqCli.EndToEnd/SeqCli.EndToEnd.csproj +++ b/test/SeqCli.EndToEnd/SeqCli.EndToEnd.csproj @@ -2,7 +2,7 @@ Exe - net8.0;net8.0-windows + net8.0 diff --git a/test/SeqCli.Tests/SeqCli.Tests.csproj b/test/SeqCli.Tests/SeqCli.Tests.csproj index 30d682cc..e93f0a96 100644 --- a/test/SeqCli.Tests/SeqCli.Tests.csproj +++ b/test/SeqCli.Tests/SeqCli.Tests.csproj @@ -1,6 +1,6 @@  - net8.0;net8.0-windows + net8.0 From b1178d790ce13695c3f12a9e7465dfba0d7d4c45 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 29 Feb 2024 14:59:21 +1000 Subject: [PATCH 12/51] Merge changes from upstream --- seqcli.sln.DotSettings | 1 + .../Cli/Commands/Forwarder/RunCommand.cs | 18 ++-- src/SeqCli/Config/ConnectionConfig.cs | 12 ++- .../Config/Forwarder/ForwarderApiConfig.cs | 2 +- .../Config/Forwarder/ForwarderConfig.cs | 4 - .../Forwarder/ForwarderStorageConfig.cs | 2 +- .../Forwarder/Config/SeqForwarderApiConfig.cs | 21 ---- .../Forwarder/Config/SeqForwarderConfig.cs | 95 ------------------- .../Config/SeqForwarderDiagnosticConfig.cs | 44 --------- .../Config/SeqForwarderOutputConfig.cs | 56 ----------- .../Config/SeqForwarderStorageConfig.cs | 21 ---- .../Multiplexing/ActiveLogBufferMap.cs | 29 ++---- .../Multiplexing/HttpLogShipperFactory.cs | 6 +- src/SeqCli/Forwarder/SeqForwarderModule.cs | 15 ++- .../Forwarder/Shipper/HttpLogShipper.cs | 10 +- .../Forwarder/Web/Api/ApiRootController.cs | 4 +- .../Forwarder/Web/Api/IngestionController.cs | 6 +- .../Multiplexing/ActiveLogBufferMapTests.cs | 6 +- 18 files changed, 54 insertions(+), 298 deletions(-) delete mode 100644 src/SeqCli/Forwarder/Config/SeqForwarderApiConfig.cs delete mode 100644 src/SeqCli/Forwarder/Config/SeqForwarderConfig.cs delete mode 100644 src/SeqCli/Forwarder/Config/SeqForwarderDiagnosticConfig.cs delete mode 100644 src/SeqCli/Forwarder/Config/SeqForwarderOutputConfig.cs delete mode 100644 src/SeqCli/Forwarder/Config/SeqForwarderStorageConfig.cs diff --git a/seqcli.sln.DotSettings b/seqcli.sln.DotSettings index 6c5c917e..a31aab98 100644 --- a/seqcli.sln.DotSettings +++ b/seqcli.sln.DotSettings @@ -1,4 +1,5 @@  + IO MS True True diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index 4c39c38b..95f1772c 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -13,7 +13,6 @@ // limitations under the License. using Autofac; -using Seq.Forwarder.Config; using Serilog; using Serilog.Events; using Serilog.Formatting.Compact; @@ -30,6 +29,7 @@ using Seq.Forwarder.Web.Host; using SeqCli.Cli; using SeqCli.Cli.Features; +using SeqCli.Config; using Serilog.Core; // ReSharper disable UnusedType.Global @@ -65,29 +65,29 @@ protected override async Task Run(string[] unrecognized) Console.WriteLine(); } - SeqForwarderConfig config; + SeqCliConfig config; try { - config = SeqForwarderConfig.ReadOrInit(_storagePath.ConfigFilePath); + config = SeqCliConfig.Read(); // _storagePath.ConfigFilePath); } catch (Exception ex) { await using var logger = CreateLogger( LogEventLevel.Information, - SeqForwarderDiagnosticConfig.GetDefaultInternalLogPath()); + ForwarderDiagnosticConfig.GetDefaultInternalLogPath()); logger.Fatal(ex, "Failed to load configuration from {ConfigFilePath}", _storagePath.ConfigFilePath); return 1; } Log.Logger = CreateLogger( - config.Diagnostics.InternalLoggingLevel, - config.Diagnostics.InternalLogPath, - config.Diagnostics.InternalLogServerUri, - config.Diagnostics.InternalLogServerApiKey); + config.Forwarder.Diagnostics.InternalLoggingLevel, + config.Forwarder.Diagnostics.InternalLogPath, + config.Forwarder.Diagnostics.InternalLogServerUri, + config.Forwarder.Diagnostics.InternalLogServerApiKey); - var listenUri = _listenUri.ListenUri ?? config.Api.ListenUri; + var listenUri = _listenUri.ListenUri ?? config.Forwarder.Api.ListenUri; try { diff --git a/src/SeqCli/Config/ConnectionConfig.cs b/src/SeqCli/Config/ConnectionConfig.cs index 57b6fd50..a726ee8f 100644 --- a/src/SeqCli/Config/ConnectionConfig.cs +++ b/src/SeqCli/Config/ConnectionConfig.cs @@ -14,11 +14,12 @@ using System; using Newtonsoft.Json; +using Seq.Forwarder.Cryptography; using SeqCli.Util; namespace SeqCli.Config; -class ConnectionConfig +public class ConnectionConfig { const string ProtectedDataPrefix = "pd."; @@ -57,4 +58,13 @@ public string? ApiKey EncodedApiKey = value; } } + + public string? GetApiKey(IStringDataProtector dataProtector) + { + throw new NotImplementedException(); + } + + public uint? PooledConnectionLifetimeMilliseconds { get; set; } = null; + public ulong EventBodyLimitBytes { get; set; } = 256 * 1024; + public ulong PayloadLimitBytes { get; set; } = 10 * 1024 * 1024; } \ No newline at end of file diff --git a/src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs index 321e8017..a056588e 100644 --- a/src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs +++ b/src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs @@ -2,5 +2,5 @@ namespace SeqCli.Config; class ForwarderApiConfig { - public string ListenUri { get; set; } = "http://localhost:15341"; + public string ListenUri { get; set; } = "http://127.0.0.1:15341"; } \ No newline at end of file diff --git a/src/SeqCli/Config/Forwarder/ForwarderConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderConfig.cs index 621072cc..9cc2a0db 100644 --- a/src/SeqCli/Config/Forwarder/ForwarderConfig.cs +++ b/src/SeqCli/Config/Forwarder/ForwarderConfig.cs @@ -2,10 +2,6 @@ namespace SeqCli.Config; class ForwarderConfig { - public uint? PooledConnectionLifetimeMilliseconds { get; set; } = null; - public ulong EventBodyLimitBytes { get; set; } = 256 * 1024; - public ulong PayloadLimitBytes { get; set; } = 10 * 1024 * 1024; - public ForwarderStorageConfig Storage { get; set; } = new(); public ForwarderDiagnosticConfig Diagnostics { get; set; } = new(); public ForwarderApiConfig Api { get; set; } = new(); diff --git a/src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs index f2143eaa..e56ab52d 100644 --- a/src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs +++ b/src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs @@ -2,5 +2,5 @@ namespace SeqCli.Config; public class ForwarderStorageConfig { - public int BufferSizeBytes { get; set; } = 67_108_864; + public ulong BufferSizeBytes { get; set; } = 67_108_864; } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Config/SeqForwarderApiConfig.cs b/src/SeqCli/Forwarder/Config/SeqForwarderApiConfig.cs deleted file mode 100644 index d2e0aaaa..00000000 --- a/src/SeqCli/Forwarder/Config/SeqForwarderApiConfig.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2016-2017 Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -namespace Seq.Forwarder.Config -{ - class SeqForwarderApiConfig - { - public string ListenUri { get; set; } = "http://localhost:15341"; - } -} diff --git a/src/SeqCli/Forwarder/Config/SeqForwarderConfig.cs b/src/SeqCli/Forwarder/Config/SeqForwarderConfig.cs deleted file mode 100644 index d39abb59..00000000 --- a/src/SeqCli/Forwarder/Config/SeqForwarderConfig.cs +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2016-2017 Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.IO; -using System.Linq; -using System.Reflection; -using Microsoft.Extensions.Configuration; -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; -using Newtonsoft.Json.Serialization; - -// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global - -namespace Seq.Forwarder.Config -{ - class SeqForwarderConfig - { - static JsonSerializerSettings SerializerSettings { get; } = new JsonSerializerSettings - { - ContractResolver = new CamelCasePropertyNamesContractResolver(), - Converters = - { - new StringEnumConverter() - } - }; - - public static SeqForwarderConfig ReadOrInit(string filename, bool includeEnvironmentVariables = true) - { - if (filename == null) throw new ArgumentNullException(nameof(filename)); - - if (!File.Exists(filename)) - { - var config = new SeqForwarderConfig(); - Write(filename, config); - return config; - } - - var content = File.ReadAllText(filename); - var combinedConfig = JsonConvert.DeserializeObject(content, SerializerSettings) - ?? throw new ArgumentException("Configuration content is null."); - - if (includeEnvironmentVariables) - { - // Any Environment Variables overwrite those in the Config File - var envVarConfig = new ConfigurationBuilder().AddEnvironmentVariables("FORWARDER_").Build(); - foreach (var sectionProperty in typeof(SeqForwarderConfig).GetTypeInfo().DeclaredProperties - .Where(p => p.GetMethod != null && p.GetMethod.IsPublic && !p.GetMethod.IsStatic)) - { - foreach (var subGroupProperty in sectionProperty.PropertyType.GetTypeInfo().DeclaredProperties - .Where(p => p.GetMethod != null && p.GetMethod.IsPublic && p.SetMethod != null && p.SetMethod.IsPublic && !p.GetMethod.IsStatic)) - { - var envVarName = sectionProperty.Name.ToUpper() + "_" + subGroupProperty.Name.ToUpper(); - var envVarVal = envVarConfig.GetValue(subGroupProperty.PropertyType, envVarName); - if (envVarVal != null) - { - subGroupProperty.SetValue(sectionProperty.GetValue(combinedConfig), envVarVal); - } - } - } - } - - return combinedConfig; - } - - public static void Write(string filename, SeqForwarderConfig data) - { - if (filename == null) throw new ArgumentNullException(nameof(filename)); - if (data == null) throw new ArgumentNullException(nameof(data)); - - var dir = Path.GetDirectoryName(filename); - if (!Directory.Exists(dir)) - Directory.CreateDirectory(dir!); - - var content = JsonConvert.SerializeObject(data, Formatting.Indented, SerializerSettings); - File.WriteAllText(filename, content); - } - - public SeqForwarderDiagnosticConfig Diagnostics { get; set; } = new SeqForwarderDiagnosticConfig(); - public SeqForwarderOutputConfig Output { get; set; } = new SeqForwarderOutputConfig(); - public SeqForwarderStorageConfig Storage { get; set; } = new SeqForwarderStorageConfig(); - public SeqForwarderApiConfig Api { get; set; } = new SeqForwarderApiConfig(); - } -} diff --git a/src/SeqCli/Forwarder/Config/SeqForwarderDiagnosticConfig.cs b/src/SeqCli/Forwarder/Config/SeqForwarderDiagnosticConfig.cs deleted file mode 100644 index d1bca9f3..00000000 --- a/src/SeqCli/Forwarder/Config/SeqForwarderDiagnosticConfig.cs +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2016-2017 Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.IO; -using Serilog.Events; - -namespace Seq.Forwarder.Config -{ - public class SeqForwarderDiagnosticConfig - { - public string InternalLogPath { get; set; } = GetDefaultInternalLogPath(); - public LogEventLevel InternalLoggingLevel { get; set; } = LogEventLevel.Information; - public string? InternalLogServerUri { get; set; } - public string? InternalLogServerApiKey { get; set; } - public bool IngestionLogShowDetail { get; set; } - - public static string GetDefaultInternalLogPath() - { - return Path.Combine( -#if WINDOWS - // Common, here, because the service may run as Local Service, which has no obvious home - // directory. - Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), -#else - // Specific to and writable by the current user. - Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), -#endif - @"Seq", - "Logs"); - } - } -} diff --git a/src/SeqCli/Forwarder/Config/SeqForwarderOutputConfig.cs b/src/SeqCli/Forwarder/Config/SeqForwarderOutputConfig.cs deleted file mode 100644 index a48bdf76..00000000 --- a/src/SeqCli/Forwarder/Config/SeqForwarderOutputConfig.cs +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using Newtonsoft.Json; -using Seq.Forwarder.Cryptography; - -// ReSharper disable UnusedMember.Global, AutoPropertyCanBeMadeGetOnly.Global - -namespace Seq.Forwarder.Config -{ - public class SeqForwarderOutputConfig - { - public string ServerUrl { get; set; } = "http://localhost:5341"; - public ulong EventBodyLimitBytes { get; set; } = 256 * 1024; - public ulong RawPayloadLimitBytes { get; set; } = 10 * 1024 * 1024; - public uint? PooledConnectionLifetimeMilliseconds { get; set; } = null; - - const string ProtectedDataPrefix = "pd."; - - public string? ApiKey { get; set; } - - public string? GetApiKey(IStringDataProtector dataProtector) - { - if (string.IsNullOrWhiteSpace(ApiKey)) - return null; - - if (!ApiKey.StartsWith(ProtectedDataPrefix)) - return ApiKey; - - return dataProtector.Unprotect(ApiKey.Substring(ProtectedDataPrefix.Length)); - } - - public void SetApiKey(string? apiKey, IStringDataProtector dataProtector) - { - if (string.IsNullOrWhiteSpace(apiKey)) - { - ApiKey = null; - return; - } - - ApiKey = $"{ProtectedDataPrefix}{dataProtector.Protect(apiKey)}"; - } - } -} diff --git a/src/SeqCli/Forwarder/Config/SeqForwarderStorageConfig.cs b/src/SeqCli/Forwarder/Config/SeqForwarderStorageConfig.cs deleted file mode 100644 index 2f713b7d..00000000 --- a/src/SeqCli/Forwarder/Config/SeqForwarderStorageConfig.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2016-2017 Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -namespace Seq.Forwarder.Config -{ - public class SeqForwarderStorageConfig - { - public ulong BufferSizeBytes { get; set; } = 64 * 1024 * 1024; - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs index 13779570..d87851e5 100644 --- a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs +++ b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs @@ -16,10 +16,10 @@ using System.Collections.Generic; using System.IO; using System.Net; -using Seq.Forwarder.Config; using Seq.Forwarder.Cryptography; using Seq.Forwarder.Storage; using Seq.Forwarder.Web; +using SeqCli.Config; using Serilog; namespace Seq.Forwarder.Multiplexing @@ -29,26 +29,26 @@ public class ActiveLogBufferMap : IDisposable const string DataFileName = "data.mdb", LockFileName = "lock.mdb", ApiKeyFileName = ".apikey"; readonly ulong _bufferSizeBytes; - readonly SeqForwarderOutputConfig _outputConfig; + readonly ConnectionConfig _connectionConfig; readonly ILogShipperFactory _shipperFactory; readonly IStringDataProtector _dataProtector; readonly string _bufferPath; readonly ILogger _log = Log.ForContext(); - readonly object _sync = new object(); + readonly object _sync = new(); bool _loaded; ActiveLogBuffer? _noApiKeyLogBuffer; readonly Dictionary _buffersByApiKey = new Dictionary(); public ActiveLogBufferMap( string bufferPath, - SeqForwarderStorageConfig storageConfig, - SeqForwarderOutputConfig outputConfig, + ForwarderStorageConfig storageConfig, + ConnectionConfig outputConfig, ILogShipperFactory logShipperFactory, IStringDataProtector dataProtector) { _bufferSizeBytes = storageConfig.BufferSizeBytes; - _outputConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); + _connectionConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); _shipperFactory = logShipperFactory ?? throw new ArgumentNullException(nameof(logShipperFactory)); _dataProtector = dataProtector ?? throw new ArgumentNullException(nameof(dataProtector)); _bufferPath = bufferPath ?? throw new ArgumentNullException(nameof(bufferPath)); @@ -85,7 +85,7 @@ public void Load() } else { - _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _outputConfig.GetApiKey(_dataProtector))); + _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _connectionConfig.GetApiKey(_dataProtector))); } } @@ -158,7 +158,7 @@ public LogBuffer GetLogBuffer(string? apiKey) { _log.Information("Creating a new default log buffer in {Path}", _bufferPath); var buffer = new LogBuffer(_bufferPath, _bufferSizeBytes); - _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _outputConfig.GetApiKey(_dataProtector))); + _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _connectionConfig.GetApiKey(_dataProtector))); _noApiKeyLogBuffer.Shipper.Start(); } return _noApiKeyLogBuffer.Buffer; @@ -190,19 +190,6 @@ public void Dispose() } } - public void Enumerate(Action action) - { - if (action == null) throw new ArgumentNullException(nameof(action)); - - lock (_sync) - { - foreach (var buffer in OpenBuffers) - { - buffer.Buffer.Enumerate(action); - } - } - } - public static void Truncate(string bufferPath) { DeleteIfExists(Path.Combine(bufferPath, DataFileName)); diff --git a/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs index 4de973ad..ce546c0e 100644 --- a/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs +++ b/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs @@ -14,9 +14,9 @@ using System; using System.Net.Http; -using Seq.Forwarder.Config; using Seq.Forwarder.Shipper; using Seq.Forwarder.Storage; +using SeqCli.Config; namespace Seq.Forwarder.Multiplexing { @@ -24,9 +24,9 @@ class HttpLogShipperFactory : ILogShipperFactory { readonly HttpClient _outputHttpClient; readonly ServerResponseProxy _serverResponseProxy; - readonly SeqForwarderOutputConfig _outputConfig; + readonly ConnectionConfig _outputConfig; - public HttpLogShipperFactory(ServerResponseProxy serverResponseProxy, SeqForwarderOutputConfig outputConfig, HttpClient outputHttpClient) + public HttpLogShipperFactory(ServerResponseProxy serverResponseProxy, ConnectionConfig outputConfig, HttpClient outputHttpClient) { _outputHttpClient = outputHttpClient; _serverResponseProxy = serverResponseProxy ?? throw new ArgumentNullException(nameof(serverResponseProxy)); diff --git a/src/SeqCli/Forwarder/SeqForwarderModule.cs b/src/SeqCli/Forwarder/SeqForwarderModule.cs index d8323510..5858334b 100644 --- a/src/SeqCli/Forwarder/SeqForwarderModule.cs +++ b/src/SeqCli/Forwarder/SeqForwarderModule.cs @@ -16,19 +16,19 @@ using System.Net.Http; using System.Threading; using Autofac; -using Seq.Forwarder.Config; using Seq.Forwarder.Cryptography; using Seq.Forwarder.Multiplexing; using Seq.Forwarder.Web.Host; +using SeqCli.Config; namespace Seq.Forwarder { class SeqForwarderModule : Module { readonly string _bufferPath; - readonly SeqForwarderConfig _config; + readonly SeqCliConfig _config; - public SeqForwarderModule(string bufferPath, SeqForwarderConfig config) + public SeqForwarderModule(string bufferPath, SeqCliConfig config) { _bufferPath = bufferPath ?? throw new ArgumentNullException(nameof(bufferPath)); _config = config ?? throw new ArgumentNullException(nameof(config)); @@ -46,7 +46,7 @@ protected override void Load(ContainerBuilder builder) builder.Register(c => { - var outputConfig = c.Resolve(); + var outputConfig = c.Resolve(); var baseUri = outputConfig.ServerUrl; if (string.IsNullOrWhiteSpace(baseUri)) throw new ArgumentException("The destination Seq server URL must be configured in SeqForwarder.json."); @@ -77,10 +77,9 @@ protected override void Load(ContainerBuilder builder) builder.RegisterInstance(StringDataProtector.CreatePlatformDefault()); builder.RegisterInstance(_config); - builder.RegisterInstance(_config.Api); - builder.RegisterInstance(_config.Diagnostics); - builder.RegisterInstance(_config.Output); - builder.RegisterInstance(_config.Storage); + builder.RegisterInstance(_config.Forwarder.Api); + builder.RegisterInstance(_config.Forwarder.Diagnostics); + builder.RegisterInstance(_config.Forwarder.Storage); } } } diff --git a/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs b/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs index 5ebe7eb1..0ebb7672 100644 --- a/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs +++ b/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs @@ -19,12 +19,12 @@ using System.Net.Http.Headers; using System.Text; using System.Threading; -using Seq.Forwarder.Config; using Seq.Forwarder.Storage; using Serilog; using System.Threading.Tasks; using Seq.Forwarder.Multiplexing; using Seq.Forwarder.Util; +using SeqCli.Config; namespace Seq.Forwarder.Shipper { @@ -34,7 +34,7 @@ sealed class HttpLogShipper : LogShipper readonly string? _apiKey; readonly LogBuffer _logBuffer; - readonly SeqForwarderOutputConfig _outputConfig; + readonly ConnectionConfig _outputConfig; readonly HttpClient _httpClient; readonly ExponentialBackoffConnectionSchedule _connectionSchedule; readonly ServerResponseProxy _serverResponseProxy; @@ -48,7 +48,7 @@ sealed class HttpLogShipper : LogShipper static readonly TimeSpan QuietWaitPeriod = TimeSpan.FromSeconds(2), MaximumConnectionInterval = TimeSpan.FromMinutes(2); - public HttpLogShipper(LogBuffer logBuffer, string? apiKey, SeqForwarderOutputConfig outputConfig, ServerResponseProxy serverResponseProxy, HttpClient outputHttpClient) + public HttpLogShipper(LogBuffer logBuffer, string? apiKey, ConnectionConfig outputConfig, ServerResponseProxy serverResponseProxy, HttpClient outputHttpClient) { _apiKey = apiKey; _httpClient = outputHttpClient ?? throw new ArgumentNullException(nameof(outputHttpClient)); @@ -117,7 +117,7 @@ async Task OnTickAsync() var sendingSingles = 0; do { - var available = _logBuffer.Peek((int)_outputConfig.RawPayloadLimitBytes); + var available = _logBuffer.Peek((int)_outputConfig.PayloadLimitBytes); if (available.Length == 0) { if (DateTime.UtcNow < _nextRequiredLevelCheck || _connectionSchedule.LastConnectionFailed) @@ -212,7 +212,7 @@ void MakePayload(LogBufferEntry[] entries, bool oneOnly, out Stream utf8Payload, var content = new StreamWriter(raw, Encoding.UTF8); content.Write("{\"Events\":["); content.Flush(); - var contentRemainingBytes = (int) _outputConfig.RawPayloadLimitBytes - 13; // Includes closing delims + var contentRemainingBytes = (int) _outputConfig.PayloadLimitBytes - 13; // Includes closing delims var delimStart = ""; foreach (var logBufferEntry in entries) diff --git a/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs b/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs index e688faac..1d0e44eb 100644 --- a/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs +++ b/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs @@ -15,8 +15,8 @@ using System.IO; using System.Text; using Microsoft.AspNetCore.Mvc; -using Seq.Forwarder.Config; using Seq.Forwarder.Diagnostics; +using SeqCli.Config; using Serilog.Formatting.Display; namespace Seq.Forwarder.Web.Api @@ -26,7 +26,7 @@ public class ApiRootController : Controller static readonly Encoding Encoding = new UTF8Encoding(false); readonly MessageTemplateTextFormatter _ingestionLogFormatter; - public ApiRootController(SeqForwarderDiagnosticConfig diagnosticConfig) + public ApiRootController(ForwarderDiagnosticConfig diagnosticConfig) { var template = "[{Timestamp:o} {Level:u3}] {Message}{NewLine}"; if (diagnosticConfig.IngestionLogShowDetail) diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionController.cs b/src/SeqCli/Forwarder/Web/Api/IngestionController.cs index 76fbbc11..7be6b9db 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionController.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionController.cs @@ -24,11 +24,11 @@ using Microsoft.Net.Http.Headers; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using Seq.Forwarder.Config; using Seq.Forwarder.Diagnostics; using Seq.Forwarder.Multiplexing; using Seq.Forwarder.Schema; using Seq.Forwarder.Shipper; +using SeqCli.Config; namespace Seq.Forwarder.Web.Api { @@ -38,13 +38,13 @@ public class IngestionController : Controller const string ClefMediaType = "application/vnd.serilog.clef"; readonly ActiveLogBufferMap _logBufferMap; - readonly SeqForwarderOutputConfig _outputConfig; + readonly ConnectionConfig _outputConfig; readonly ServerResponseProxy _serverResponseProxy; readonly JsonSerializer _rawSerializer = JsonSerializer.Create( new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); - public IngestionController(ActiveLogBufferMap logBufferMap, SeqForwarderOutputConfig outputConfig, ServerResponseProxy serverResponseProxy) + public IngestionController(ActiveLogBufferMap logBufferMap, ConnectionConfig outputConfig, ServerResponseProxy serverResponseProxy) { _logBufferMap = logBufferMap; _outputConfig = outputConfig; diff --git a/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs index 46188948..3a330002 100644 --- a/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs +++ b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs @@ -1,9 +1,9 @@ using System.IO; using System.Linq; -using Seq.Forwarder.Config; using Seq.Forwarder.Cryptography; using Seq.Forwarder.Multiplexing; using Seq.Forwarder.Tests.Support; +using SeqCli.Config; using SeqCli.Tests.Support; using Xunit; @@ -74,8 +74,8 @@ public void EntriesSurviveReloads() static ActiveLogBufferMap CreateActiveLogBufferMap(TempFolder tmp) { - var config = new SeqForwarderConfig(); - var map = new ActiveLogBufferMap(tmp.Path, config.Storage, config.Output, new InertLogShipperFactory(), StringDataProtector.CreatePlatformDefault()); + var config = new SeqCliConfig(); + var map = new ActiveLogBufferMap(tmp.Path, config.Forwarder.Storage, config.Connection, new InertLogShipperFactory(), StringDataProtector.CreatePlatformDefault()); map.Load(); return map; } From e07420cff7c0026b359c285c81425e4d814a642b Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 29 Feb 2024 15:00:02 +1000 Subject: [PATCH 13/51] Adjust namespaces --- src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs | 17 +++++++++-------- .../Cli/Commands/Forwarder/TruncateCommand.cs | 6 ++---- src/SeqCli/Config/ConnectionConfig.cs | 2 +- .../Config/Forwarder/ForwarderApiConfig.cs | 2 +- src/SeqCli/Config/Forwarder/ForwarderConfig.cs | 2 +- .../Forwarder/ForwarderDiagnosticConfig.cs | 2 +- .../Config/Forwarder/ForwarderStorageConfig.cs | 2 +- src/SeqCli/Config/SeqCliConfig.cs | 1 + .../Cryptography/IStringDataProtector.cs | 2 +- .../Cryptography/StringDataProtector.cs | 2 +- .../Cryptography/UnprotectedStringData.cs | 2 +- .../Forwarder/Diagnostics/InMemorySink.cs | 2 +- .../Forwarder/Diagnostics/IngestionLog.cs | 2 +- .../Forwarder/Multiplexing/ActiveLogBuffer.cs | 6 +++--- .../Multiplexing/ActiveLogBufferMap.cs | 9 +++++---- .../Multiplexing/HttpLogShipperFactory.cs | 6 +++--- .../Multiplexing/ILogShipperFactory.cs | 6 +++--- .../Multiplexing/InertLogShipperFactory.cs | 6 +++--- .../Multiplexing/ServerResponseProxy.cs | 2 +- src/SeqCli/Forwarder/Schema/EventSchema.cs | 4 ++-- src/SeqCli/Forwarder/SeqForwarderModule.cs | 8 ++++---- .../ExponentialBackoffConnectionSchedule.cs | 2 +- src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs | 10 +++++----- src/SeqCli/Forwarder/Shipper/InertLogShipper.cs | 2 +- src/SeqCli/Forwarder/Shipper/LogShipper.cs | 2 +- src/SeqCli/Forwarder/Shipper/SeqApi.cs | 2 +- src/SeqCli/Forwarder/Storage/LogBuffer.cs | 2 +- src/SeqCli/Forwarder/Storage/LogBufferEntry.cs | 2 +- src/SeqCli/Forwarder/Util/CaptiveProcess.cs | 2 +- .../Forwarder/Util/EnumerableExtensions.cs | 2 +- .../Forwarder/Util/ExecutionEnvironment.cs | 2 +- .../Forwarder/Util/UnclosableStreamWrapper.cs | 2 +- .../Forwarder/Web/Api/ApiRootController.cs | 6 +++--- .../Forwarder/Web/Api/IngestionController.cs | 10 +++++----- src/SeqCli/Forwarder/Web/Host/ServerService.cs | 6 +++--- src/SeqCli/Forwarder/Web/Host/Startup.cs | 2 +- .../Forwarder/Web/RequestProcessingException.cs | 2 +- .../RetentionPolicyBasicsTestCase.cs | 3 +-- .../Multiplexing/ActiveLogBufferMapTests.cs | 7 +++---- .../Forwarder/Schema/EventSchemaTests.cs | 4 ++-- .../Shipper/ServerResponseProxyTests.cs | 6 ++---- .../Forwarder/Storage/LogBufferTests.cs | 5 ++--- test/SeqCli.Tests/Support/TempFolder.cs | 8 ++++---- 43 files changed, 88 insertions(+), 92 deletions(-) diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index 95f1772c..ca81665b 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -12,29 +12,30 @@ // See the License for the specific language governing permissions and // limitations under the License. -using Autofac; -using Serilog; -using Serilog.Events; -using Serilog.Formatting.Compact; using System; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net; using System.Security.Cryptography.X509Certificates; using System.Threading.Tasks; +using Autofac; using Autofac.Extensions.DependencyInjection; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Hosting; -using Seq.Forwarder.Util; -using Seq.Forwarder.Web.Host; -using SeqCli.Cli; using SeqCli.Cli.Features; using SeqCli.Config; +using SeqCli.Config.Forwarder; +using SeqCli.Forwarder; +using SeqCli.Forwarder.Util; +using SeqCli.Forwarder.Web.Host; +using Serilog; using Serilog.Core; +using Serilog.Events; +using Serilog.Formatting.Compact; // ReSharper disable UnusedType.Global -namespace Seq.Forwarder.Cli.Commands +namespace SeqCli.Cli.Commands.Forwarder { [Command("forwarder", "run", "Run the server interactively")] class RunCommand : Command diff --git a/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs index baf0320c..856c8f4d 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs @@ -13,14 +13,12 @@ // limitations under the License. using System; -using System.IO; using System.Threading.Tasks; -using Seq.Forwarder.Multiplexing; -using SeqCli.Cli; using SeqCli.Cli.Features; +using SeqCli.Forwarder.Multiplexing; using Serilog; -namespace Seq.Forwarder.Cli.Commands +namespace SeqCli.Cli.Commands.Forwarder { [Command("forwarder", "truncate", "Clear the log buffer contents")] class TruncateCommand : Command diff --git a/src/SeqCli/Config/ConnectionConfig.cs b/src/SeqCli/Config/ConnectionConfig.cs index a726ee8f..821b6ad6 100644 --- a/src/SeqCli/Config/ConnectionConfig.cs +++ b/src/SeqCli/Config/ConnectionConfig.cs @@ -14,7 +14,7 @@ using System; using Newtonsoft.Json; -using Seq.Forwarder.Cryptography; +using SeqCli.Forwarder.Cryptography; using SeqCli.Util; namespace SeqCli.Config; diff --git a/src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs index a056588e..3c0291a6 100644 --- a/src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs +++ b/src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs @@ -1,4 +1,4 @@ -namespace SeqCli.Config; +namespace SeqCli.Config.Forwarder; class ForwarderApiConfig { diff --git a/src/SeqCli/Config/Forwarder/ForwarderConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderConfig.cs index 9cc2a0db..fac69ad4 100644 --- a/src/SeqCli/Config/Forwarder/ForwarderConfig.cs +++ b/src/SeqCli/Config/Forwarder/ForwarderConfig.cs @@ -1,4 +1,4 @@ -namespace SeqCli.Config; +namespace SeqCli.Config.Forwarder; class ForwarderConfig { diff --git a/src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs index 8366a6e4..1edc44fe 100644 --- a/src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs +++ b/src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs @@ -2,7 +2,7 @@ using System.IO; using Serilog.Events; -namespace SeqCli.Config; +namespace SeqCli.Config.Forwarder; public class ForwarderDiagnosticConfig { diff --git a/src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs index e56ab52d..455daacb 100644 --- a/src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs +++ b/src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs @@ -1,4 +1,4 @@ -namespace SeqCli.Config; +namespace SeqCli.Config.Forwarder; public class ForwarderStorageConfig { diff --git a/src/SeqCli/Config/SeqCliConfig.cs b/src/SeqCli/Config/SeqCliConfig.cs index 8ff11a08..93160e69 100644 --- a/src/SeqCli/Config/SeqCliConfig.cs +++ b/src/SeqCli/Config/SeqCliConfig.cs @@ -18,6 +18,7 @@ using Newtonsoft.Json; using Newtonsoft.Json.Converters; using Newtonsoft.Json.Serialization; +using SeqCli.Config.Forwarder; namespace SeqCli.Config; diff --git a/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs b/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs index 24ef61b0..0d49b92e 100644 --- a/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs +++ b/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs @@ -1,4 +1,4 @@ -namespace Seq.Forwarder.Cryptography +namespace SeqCli.Forwarder.Cryptography { public interface IStringDataProtector { diff --git a/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs b/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs index 64ef755c..46bd1ae2 100644 --- a/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs +++ b/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs @@ -1,4 +1,4 @@ -namespace Seq.Forwarder.Cryptography +namespace SeqCli.Forwarder.Cryptography { static class StringDataProtector { diff --git a/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs b/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs index b5213375..ef556083 100644 --- a/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs +++ b/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs @@ -2,7 +2,7 @@ using Serilog; -namespace Seq.Forwarder.Cryptography +namespace SeqCli.Forwarder.Cryptography { public class UnprotectedStringData : IStringDataProtector { diff --git a/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs b/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs index 00797ec0..38d0c6db 100644 --- a/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs +++ b/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs @@ -18,7 +18,7 @@ using Serilog.Core; using Serilog.Events; -namespace Seq.Forwarder.Diagnostics +namespace SeqCli.Forwarder.Diagnostics { public class InMemorySink : ILogEventSink { diff --git a/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs b/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs index 52e5af13..e2fa7b5c 100644 --- a/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs +++ b/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs @@ -18,7 +18,7 @@ using Serilog; using Serilog.Events; -namespace Seq.Forwarder.Diagnostics +namespace SeqCli.Forwarder.Diagnostics { static class IngestionLog { diff --git a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs index 637f4c09..ce0f7318 100644 --- a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs +++ b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs @@ -13,10 +13,10 @@ // limitations under the License. using System; -using Seq.Forwarder.Shipper; -using Seq.Forwarder.Storage; +using SeqCli.Forwarder.Shipper; +using SeqCli.Forwarder.Storage; -namespace Seq.Forwarder.Multiplexing +namespace SeqCli.Forwarder.Multiplexing { sealed class ActiveLogBuffer : IDisposable { diff --git a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs index d87851e5..47d19db1 100644 --- a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs +++ b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs @@ -16,13 +16,14 @@ using System.Collections.Generic; using System.IO; using System.Net; -using Seq.Forwarder.Cryptography; -using Seq.Forwarder.Storage; -using Seq.Forwarder.Web; using SeqCli.Config; +using SeqCli.Config.Forwarder; +using SeqCli.Forwarder.Cryptography; +using SeqCli.Forwarder.Storage; +using SeqCli.Forwarder.Web; using Serilog; -namespace Seq.Forwarder.Multiplexing +namespace SeqCli.Forwarder.Multiplexing { public class ActiveLogBufferMap : IDisposable { diff --git a/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs index ce546c0e..d80e91df 100644 --- a/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs +++ b/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs @@ -14,11 +14,11 @@ using System; using System.Net.Http; -using Seq.Forwarder.Shipper; -using Seq.Forwarder.Storage; using SeqCli.Config; +using SeqCli.Forwarder.Shipper; +using SeqCli.Forwarder.Storage; -namespace Seq.Forwarder.Multiplexing +namespace SeqCli.Forwarder.Multiplexing { class HttpLogShipperFactory : ILogShipperFactory { diff --git a/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs index 554324de..85ff8df7 100644 --- a/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs +++ b/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -using Seq.Forwarder.Shipper; -using Seq.Forwarder.Storage; +using SeqCli.Forwarder.Shipper; +using SeqCli.Forwarder.Storage; -namespace Seq.Forwarder.Multiplexing +namespace SeqCli.Forwarder.Multiplexing { public interface ILogShipperFactory { diff --git a/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs index f0dd9e44..4d9c2b78 100644 --- a/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs +++ b/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -using Seq.Forwarder.Shipper; -using Seq.Forwarder.Storage; +using SeqCli.Forwarder.Shipper; +using SeqCli.Forwarder.Storage; -namespace Seq.Forwarder.Multiplexing +namespace SeqCli.Forwarder.Multiplexing { class InertLogShipperFactory : ILogShipperFactory { diff --git a/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs b/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs index 86ccc768..9c629498 100644 --- a/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs +++ b/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs @@ -14,7 +14,7 @@ using System.Collections.Generic; -namespace Seq.Forwarder.Multiplexing +namespace SeqCli.Forwarder.Multiplexing { public class ServerResponseProxy { diff --git a/src/SeqCli/Forwarder/Schema/EventSchema.cs b/src/SeqCli/Forwarder/Schema/EventSchema.cs index aa6c6f6c..104a08d3 100644 --- a/src/SeqCli/Forwarder/Schema/EventSchema.cs +++ b/src/SeqCli/Forwarder/Schema/EventSchema.cs @@ -4,10 +4,10 @@ using System.Globalization; using System.Linq; using Newtonsoft.Json.Linq; +using SeqCli.Forwarder.Util; using Serilog.Parsing; -using Seq.Forwarder.Util; -namespace Seq.Forwarder.Schema +namespace SeqCli.Forwarder.Schema { static class EventSchema { diff --git a/src/SeqCli/Forwarder/SeqForwarderModule.cs b/src/SeqCli/Forwarder/SeqForwarderModule.cs index 5858334b..319c3736 100644 --- a/src/SeqCli/Forwarder/SeqForwarderModule.cs +++ b/src/SeqCli/Forwarder/SeqForwarderModule.cs @@ -16,12 +16,12 @@ using System.Net.Http; using System.Threading; using Autofac; -using Seq.Forwarder.Cryptography; -using Seq.Forwarder.Multiplexing; -using Seq.Forwarder.Web.Host; using SeqCli.Config; +using SeqCli.Forwarder.Cryptography; +using SeqCli.Forwarder.Multiplexing; +using SeqCli.Forwarder.Web.Host; -namespace Seq.Forwarder +namespace SeqCli.Forwarder { class SeqForwarderModule : Module { diff --git a/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs b/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs index 84c32f6c..96c9c7a4 100644 --- a/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs +++ b/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs @@ -14,7 +14,7 @@ using System; -namespace Seq.Forwarder.Shipper +namespace SeqCli.Forwarder.Shipper { class ExponentialBackoffConnectionSchedule { diff --git a/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs b/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs index 0ebb7672..41354e6d 100644 --- a/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs +++ b/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs @@ -19,14 +19,14 @@ using System.Net.Http.Headers; using System.Text; using System.Threading; -using Seq.Forwarder.Storage; -using Serilog; using System.Threading.Tasks; -using Seq.Forwarder.Multiplexing; -using Seq.Forwarder.Util; using SeqCli.Config; +using SeqCli.Forwarder.Multiplexing; +using SeqCli.Forwarder.Storage; +using SeqCli.Forwarder.Util; +using Serilog; -namespace Seq.Forwarder.Shipper +namespace SeqCli.Forwarder.Shipper { sealed class HttpLogShipper : LogShipper { diff --git a/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs b/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs index 164a2939..4288b670 100644 --- a/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs +++ b/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -namespace Seq.Forwarder.Shipper +namespace SeqCli.Forwarder.Shipper { class InertLogShipper : LogShipper { diff --git a/src/SeqCli/Forwarder/Shipper/LogShipper.cs b/src/SeqCli/Forwarder/Shipper/LogShipper.cs index ac8f5157..82594955 100644 --- a/src/SeqCli/Forwarder/Shipper/LogShipper.cs +++ b/src/SeqCli/Forwarder/Shipper/LogShipper.cs @@ -14,7 +14,7 @@ using System; -namespace Seq.Forwarder.Shipper +namespace SeqCli.Forwarder.Shipper { public abstract class LogShipper : IDisposable { diff --git a/src/SeqCli/Forwarder/Shipper/SeqApi.cs b/src/SeqCli/Forwarder/Shipper/SeqApi.cs index 330dc3b8..83333823 100644 --- a/src/SeqCli/Forwarder/Shipper/SeqApi.cs +++ b/src/SeqCli/Forwarder/Shipper/SeqApi.cs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -namespace Seq.Forwarder.Shipper +namespace SeqCli.Forwarder.Shipper { static class SeqApi { diff --git a/src/SeqCli/Forwarder/Storage/LogBuffer.cs b/src/SeqCli/Forwarder/Storage/LogBuffer.cs index bac905ea..5983df2c 100644 --- a/src/SeqCli/Forwarder/Storage/LogBuffer.cs +++ b/src/SeqCli/Forwarder/Storage/LogBuffer.cs @@ -16,7 +16,7 @@ using System.Collections.Generic; using Serilog; -namespace Seq.Forwarder.Storage +namespace SeqCli.Forwarder.Storage { public class LogBuffer : IDisposable { diff --git a/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs b/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs index 464a7175..fb0851a2 100644 --- a/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs +++ b/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs @@ -14,7 +14,7 @@ // ReSharper disable InconsistentNaming -namespace Seq.Forwarder.Storage +namespace SeqCli.Forwarder.Storage { public struct LogBufferEntry { diff --git a/src/SeqCli/Forwarder/Util/CaptiveProcess.cs b/src/SeqCli/Forwarder/Util/CaptiveProcess.cs index dc12482e..eea47c83 100644 --- a/src/SeqCli/Forwarder/Util/CaptiveProcess.cs +++ b/src/SeqCli/Forwarder/Util/CaptiveProcess.cs @@ -16,7 +16,7 @@ using System.Diagnostics; using System.Threading; -namespace Seq.Forwarder.Util +namespace SeqCli.Forwarder.Util { public static class CaptiveProcess { diff --git a/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs b/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs index 612bc684..da930090 100644 --- a/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs +++ b/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace Seq.Forwarder.Util +namespace SeqCli.Forwarder.Util { static class EnumerableExtensions { diff --git a/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs b/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs index 9e94295a..26535bbf 100644 --- a/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs +++ b/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs @@ -1,4 +1,4 @@ -namespace Seq.Forwarder.Util +namespace SeqCli.Forwarder.Util { static class ExecutionEnvironment { diff --git a/src/SeqCli/Forwarder/Util/UnclosableStreamWrapper.cs b/src/SeqCli/Forwarder/Util/UnclosableStreamWrapper.cs index ce86ea12..3781a124 100644 --- a/src/SeqCli/Forwarder/Util/UnclosableStreamWrapper.cs +++ b/src/SeqCli/Forwarder/Util/UnclosableStreamWrapper.cs @@ -15,7 +15,7 @@ using System; using System.IO; -namespace Seq.Forwarder.Util +namespace SeqCli.Forwarder.Util { class UnclosableStreamWrapper : Stream { diff --git a/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs b/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs index 1d0e44eb..f295aa58 100644 --- a/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs +++ b/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs @@ -15,11 +15,11 @@ using System.IO; using System.Text; using Microsoft.AspNetCore.Mvc; -using Seq.Forwarder.Diagnostics; -using SeqCli.Config; +using SeqCli.Config.Forwarder; +using SeqCli.Forwarder.Diagnostics; using Serilog.Formatting.Display; -namespace Seq.Forwarder.Web.Api +namespace SeqCli.Forwarder.Web.Api { public class ApiRootController : Controller { diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionController.cs b/src/SeqCli/Forwarder/Web/Api/IngestionController.cs index 7be6b9db..a9ac94bb 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionController.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionController.cs @@ -24,13 +24,13 @@ using Microsoft.Net.Http.Headers; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using Seq.Forwarder.Diagnostics; -using Seq.Forwarder.Multiplexing; -using Seq.Forwarder.Schema; -using Seq.Forwarder.Shipper; using SeqCli.Config; +using SeqCli.Forwarder.Diagnostics; +using SeqCli.Forwarder.Multiplexing; +using SeqCli.Forwarder.Schema; +using SeqCli.Forwarder.Shipper; -namespace Seq.Forwarder.Web.Api +namespace SeqCli.Forwarder.Web.Api { public class IngestionController : Controller { diff --git a/src/SeqCli/Forwarder/Web/Host/ServerService.cs b/src/SeqCli/Forwarder/Web/Host/ServerService.cs index 98d0d5dc..509d86e6 100644 --- a/src/SeqCli/Forwarder/Web/Host/ServerService.cs +++ b/src/SeqCli/Forwarder/Web/Host/ServerService.cs @@ -14,11 +14,11 @@ using System; using Microsoft.Extensions.Hosting; -using Seq.Forwarder.Diagnostics; -using Seq.Forwarder.Multiplexing; +using SeqCli.Forwarder.Diagnostics; +using SeqCli.Forwarder.Multiplexing; using Serilog; -namespace Seq.Forwarder.Web.Host +namespace SeqCli.Forwarder.Web.Host { class ServerService { diff --git a/src/SeqCli/Forwarder/Web/Host/Startup.cs b/src/SeqCli/Forwarder/Web/Host/Startup.cs index 57379a33..56d6fe9f 100644 --- a/src/SeqCli/Forwarder/Web/Host/Startup.cs +++ b/src/SeqCli/Forwarder/Web/Host/Startup.cs @@ -2,7 +2,7 @@ using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.DependencyInjection; -namespace Seq.Forwarder.Web.Host +namespace SeqCli.Forwarder.Web.Host { class Startup { diff --git a/src/SeqCli/Forwarder/Web/RequestProcessingException.cs b/src/SeqCli/Forwarder/Web/RequestProcessingException.cs index d5968643..faa95e43 100644 --- a/src/SeqCli/Forwarder/Web/RequestProcessingException.cs +++ b/src/SeqCli/Forwarder/Web/RequestProcessingException.cs @@ -15,7 +15,7 @@ using System; using System.Net; -namespace Seq.Forwarder.Web +namespace SeqCli.Forwarder.Web { class RequestProcessingException : Exception { diff --git a/test/SeqCli.EndToEnd/RetentionPolicy/RetentionPolicyBasicsTestCase.cs b/test/SeqCli.EndToEnd/RetentionPolicy/RetentionPolicyBasicsTestCase.cs index fdb554d7..2a64a0e8 100644 --- a/test/SeqCli.EndToEnd/RetentionPolicy/RetentionPolicyBasicsTestCase.cs +++ b/test/SeqCli.EndToEnd/RetentionPolicy/RetentionPolicyBasicsTestCase.cs @@ -1,12 +1,11 @@ using System; using System.Threading.Tasks; -using System.Linq; using Seq.Api; using SeqCli.EndToEnd.Support; using Serilog; using Xunit; -namespace SeqCli.EndToEnd.RetentionPolicies; +namespace SeqCli.EndToEnd.RetentionPolicy; // ReSharper disable once UnusedType.Global public class RetentionPolicyBasicsTestCase : ICliTestCase diff --git a/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs index 3a330002..3466c41b 100644 --- a/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs +++ b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs @@ -1,13 +1,12 @@ using System.IO; using System.Linq; -using Seq.Forwarder.Cryptography; -using Seq.Forwarder.Multiplexing; -using Seq.Forwarder.Tests.Support; using SeqCli.Config; +using SeqCli.Forwarder.Cryptography; +using SeqCli.Forwarder.Multiplexing; using SeqCli.Tests.Support; using Xunit; -namespace Seq.Forwarder.Tests.Multiplexing +namespace SeqCli.Tests.Forwarder.Multiplexing { public class ActiveLogBufferMapTests { diff --git a/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs b/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs index 2032215e..05f1a7f3 100644 --- a/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs +++ b/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs @@ -1,10 +1,10 @@ using System.IO; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using Seq.Forwarder.Schema; +using SeqCli.Forwarder.Schema; using Xunit; -namespace Seq.Forwarder.Tests.Schema +namespace SeqCli.Tests.Forwarder.Schema { public class EventSchemaTests { diff --git a/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs b/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs index 1ac2db7c..217216a2 100644 --- a/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs +++ b/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs @@ -1,10 +1,8 @@ -using Seq.Forwarder.Multiplexing; -using Seq.Forwarder.Shipper; -using Seq.Forwarder.Tests.Support; +using SeqCli.Forwarder.Multiplexing; using SeqCli.Tests.Support; using Xunit; -namespace Seq.Forwarder.Tests.Shipper +namespace SeqCli.Tests.Forwarder.Shipper { public class ServerResponseProxyTests { diff --git a/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs b/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs index e6468d00..469ed4c9 100644 --- a/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs +++ b/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs @@ -1,10 +1,9 @@ using System.Collections.Generic; -using Seq.Forwarder.Storage; -using Seq.Forwarder.Tests.Support; +using SeqCli.Forwarder.Storage; using SeqCli.Tests.Support; using Xunit; -namespace Seq.Forwarder.Tests.Storage +namespace SeqCli.Tests.Forwarder.Storage { public class LogBufferTests { diff --git a/test/SeqCli.Tests/Support/TempFolder.cs b/test/SeqCli.Tests/Support/TempFolder.cs index f7d358ef..498ed571 100644 --- a/test/SeqCli.Tests/Support/TempFolder.cs +++ b/test/SeqCli.Tests/Support/TempFolder.cs @@ -1,11 +1,11 @@ -using System; +#nullable enable + +using System; using System.Diagnostics; using System.IO; using System.Runtime.CompilerServices; -#nullable enable - -namespace Seq.Forwarder.Tests.Support +namespace SeqCli.Tests.Support { class TempFolder : IDisposable { From 517c96e7c0cc67a8f8296dae572c86c3834fa50c Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 29 Feb 2024 15:00:26 +1000 Subject: [PATCH 14/51] Collection expressions --- src/Roastery/Fake/Person.cs | 12 ++++----- src/Roastery/Web/FaultInjectionMiddleware.cs | 6 ++--- src/Roastery/Web/Router.cs | 2 +- .../Apps/Definitions/AppMetadataReader.cs | 16 ++++-------- .../Cli/Commands/ApiKey/RemoveCommand.cs | 4 +-- .../Cli/Commands/AppInstance/RemoveCommand.cs | 4 +-- .../Cli/Commands/Dashboard/RemoveCommand.cs | 4 +-- src/SeqCli/Cli/Commands/Feed/RemoveCommand.cs | 4 +-- .../Commands/RetentionPolicy/ListCommand.cs | 4 +-- .../Cli/Commands/Signal/RemoveCommand.cs | 4 +-- src/SeqCli/Cli/Commands/User/RemoveCommand.cs | 4 +-- .../Cli/Commands/Workspace/RemoveCommand.cs | 4 +-- src/SeqCli/Cli/Options.cs | 26 +++++++++---------- src/SeqCli/Csv/CsvTokenizer.cs | 6 ++--- src/SeqCli/Forwarder/Schema/EventSchema.cs | 5 +--- src/SeqCli/PlainText/Framing/FrameReader.cs | 4 +-- src/SeqCli/PlainText/ReifiedProperties.cs | 5 +--- test/SeqCli.Tests/Cli/CommandLineHostTests.cs | 6 ++--- .../Multiplexing/ActiveLogBufferMapTests.cs | 4 +-- .../Forwarder/Storage/LogBufferTests.cs | 20 +++++++------- .../Signals/SignalExpressionParserTests.cs | 18 ++++++------- .../Templates/TemplateWriterTests.cs | 4 +-- 22 files changed, 77 insertions(+), 89 deletions(-) diff --git a/src/Roastery/Fake/Person.cs b/src/Roastery/Fake/Person.cs index 53394561..56c0ae0c 100644 --- a/src/Roastery/Fake/Person.cs +++ b/src/Roastery/Fake/Person.cs @@ -14,7 +14,7 @@ public Person(string? name, string? address) } static readonly string[] Forenames = - { + [ "Akeem", "Alice", "Alok", @@ -40,10 +40,10 @@ public Person(string? name, string? address) "Yoshi", "Zach", "Zeynep" - }; + ]; static readonly string[] Surnames = - { + [ "Anderson", "Alvarez", "Brookes", @@ -60,10 +60,10 @@ public Person(string? name, string? address) "Smith", "Xia", "Zheng" - }; + ]; static readonly string[] Streets = - { + [ "Lilac Road", "Lilly Street", "Carnation Street", @@ -78,7 +78,7 @@ public Person(string? name, string? address) "Trillium Creek Parkway", "Grevillea Street", "Kurrajong Street" - }; + ]; public static Person Generate() { diff --git a/src/Roastery/Web/FaultInjectionMiddleware.cs b/src/Roastery/Web/FaultInjectionMiddleware.cs index 27dae6d8..0c6f472e 100644 --- a/src/Roastery/Web/FaultInjectionMiddleware.cs +++ b/src/Roastery/Web/FaultInjectionMiddleware.cs @@ -18,15 +18,15 @@ public FaultInjectionMiddleware(ILogger logger, HttpServer next) { _logger = logger.ForContext(); _next = next; - _faults = new Func>[] - { + _faults = + [ Unauthorized, Unauthorized, Unauthorized, Timeout, Timeout, Disposed - }; + ]; } Task Unauthorized(HttpRequest request) diff --git a/src/Roastery/Web/Router.cs b/src/Roastery/Web/Router.cs index 5df8d7ed..0c9f7be9 100644 --- a/src/Roastery/Web/Router.cs +++ b/src/Roastery/Web/Router.cs @@ -63,7 +63,7 @@ public Router(IEnumerable controllers, ILogger logger) { using var _ = LogContext.PushProperty("Controller", controllerName); using var __ = LogContext.PushProperty("Action", actionName); - return (Task) method.Invoke(controller, new object[] {r})!; + return (Task) method.Invoke(controller, [r])!; }); _logger.Debug("Binding route HTTP {HttpMethod} {RouteTemplate} to action method {Controller}.{Action}()", diff --git a/src/SeqCli/Apps/Definitions/AppMetadataReader.cs b/src/SeqCli/Apps/Definitions/AppMetadataReader.cs index 8abd7be0..58269661 100644 --- a/src/SeqCli/Apps/Definitions/AppMetadataReader.cs +++ b/src/SeqCli/Apps/Definitions/AppMetadataReader.cs @@ -79,21 +79,15 @@ static Dictionary GetAvailableSettings(Type mainRe }); } - static readonly HashSet IntegerTypes = new() - { + static readonly HashSet IntegerTypes = + [ typeof(short), typeof(ushort), typeof(int), typeof(uint), typeof(long), typeof(ulong) - }; + ]; - static readonly HashSet DecimalTypes = new() - { - typeof(float), typeof(double), typeof(decimal) - }; + static readonly HashSet DecimalTypes = [typeof(float), typeof(double), typeof(decimal)]; - static readonly HashSet BooleanTypes = new() - { - typeof(bool) - }; + static readonly HashSet BooleanTypes = [typeof(bool)]; internal static AppSettingType GetSettingType(Type type) { diff --git a/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs index d10beb63..8935f178 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs @@ -48,8 +48,8 @@ protected override async Task Run() var connection = _connectionFactory.Connect(_connection); - var toRemove = _entityIdentity.Id != null ? - new[] {await connection.ApiKeys.FindAsync(_entityIdentity.Id)} : + var toRemove = _entityIdentity.Id != null ? [await connection.ApiKeys.FindAsync(_entityIdentity.Id)] + : (await connection.ApiKeys.ListAsync()) .Where(ak => _entityIdentity.Title == ak.Title) .ToArray(); diff --git a/src/SeqCli/Cli/Commands/AppInstance/RemoveCommand.cs b/src/SeqCli/Cli/Commands/AppInstance/RemoveCommand.cs index 76e90915..376f7354 100644 --- a/src/SeqCli/Cli/Commands/AppInstance/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/AppInstance/RemoveCommand.cs @@ -34,8 +34,8 @@ protected override async Task Run() var connection = _connectionFactory.Connect(_connection); - var toRemove = _entityIdentity.Id != null ? - new[] {await connection.AppInstances.FindAsync(_entityIdentity.Id)} : + var toRemove = _entityIdentity.Id != null ? [await connection.AppInstances.FindAsync(_entityIdentity.Id)] + : (await connection.AppInstances.ListAsync()) .Where(ak => _entityIdentity.Title == ak.Title) .ToArray(); diff --git a/src/SeqCli/Cli/Commands/Dashboard/RemoveCommand.cs b/src/SeqCli/Cli/Commands/Dashboard/RemoveCommand.cs index 64e45bbf..70d9a78e 100644 --- a/src/SeqCli/Cli/Commands/Dashboard/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/Dashboard/RemoveCommand.cs @@ -50,8 +50,8 @@ protected override async Task Run() var connection = _connectionFactory.Connect(_connection); - var toRemove = _entityIdentity.Id != null ? - new[] { await connection.Dashboards.FindAsync(_entityIdentity.Id) } : + var toRemove = _entityIdentity.Id != null ? [await connection.Dashboards.FindAsync(_entityIdentity.Id)] + : (await connection.Dashboards.ListAsync(ownerId: _entityOwner.OwnerId, shared: _entityOwner.IncludeShared)) .Where(dashboard => _entityIdentity.Title == dashboard.Title) .ToArray(); diff --git a/src/SeqCli/Cli/Commands/Feed/RemoveCommand.cs b/src/SeqCli/Cli/Commands/Feed/RemoveCommand.cs index 4f0ce4b4..c706fe3e 100644 --- a/src/SeqCli/Cli/Commands/Feed/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/Feed/RemoveCommand.cs @@ -58,8 +58,8 @@ protected override async Task Run() var connection = _connectionFactory.Connect(_connection); - var toRemove = _id != null ? - new[] {await connection.Feeds.FindAsync(_id)} : + var toRemove = _id != null ? [await connection.Feeds.FindAsync(_id)] + : (await connection.Feeds.ListAsync()) .Where(f => _name == f.Name) .ToArray(); diff --git a/src/SeqCli/Cli/Commands/RetentionPolicy/ListCommand.cs b/src/SeqCli/Cli/Commands/RetentionPolicy/ListCommand.cs index 57702a93..710ae634 100644 --- a/src/SeqCli/Cli/Commands/RetentionPolicy/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/RetentionPolicy/ListCommand.cs @@ -49,8 +49,8 @@ protected override async Task Run() { var connection = _connectionFactory.Connect(_connection); - var list = _id != null ? - new[] { await connection.RetentionPolicies.FindAsync(_id) } : + var list = _id != null ? [await connection.RetentionPolicies.FindAsync(_id)] + : (await connection.RetentionPolicies.ListAsync()) .ToArray(); diff --git a/src/SeqCli/Cli/Commands/Signal/RemoveCommand.cs b/src/SeqCli/Cli/Commands/Signal/RemoveCommand.cs index cf1b890e..35f3a951 100644 --- a/src/SeqCli/Cli/Commands/Signal/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/Signal/RemoveCommand.cs @@ -50,8 +50,8 @@ protected override async Task Run() var connection = _connectionFactory.Connect(_connection); - var toRemove = _entityIdentity.Id != null ? - new[] { await connection.Signals.FindAsync(_entityIdentity.Id) } : + var toRemove = _entityIdentity.Id != null ? [await connection.Signals.FindAsync(_entityIdentity.Id)] + : (await connection.Signals.ListAsync(ownerId: _entityOwner.OwnerId, shared: _entityOwner.IncludeShared)) .Where(signal => _entityIdentity.Title == signal.Title) .ToArray(); diff --git a/src/SeqCli/Cli/Commands/User/RemoveCommand.cs b/src/SeqCli/Cli/Commands/User/RemoveCommand.cs index 51b6127c..782f90d9 100644 --- a/src/SeqCli/Cli/Commands/User/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/User/RemoveCommand.cs @@ -48,8 +48,8 @@ protected override async Task Run() var connection = _connectionFactory.Connect(_connection); - var toRemove = _userIdentity.Id != null ? - new[] {await connection.Users.FindAsync(_userIdentity.Id)} : + var toRemove = _userIdentity.Id != null ? [await connection.Users.FindAsync(_userIdentity.Id)] + : (await connection.Users.ListAsync()) .Where(u => _userIdentity.Name == u.Username) .ToArray(); diff --git a/src/SeqCli/Cli/Commands/Workspace/RemoveCommand.cs b/src/SeqCli/Cli/Commands/Workspace/RemoveCommand.cs index a4990edd..470e719c 100644 --- a/src/SeqCli/Cli/Commands/Workspace/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/Workspace/RemoveCommand.cs @@ -36,8 +36,8 @@ protected override async Task Run() var connection = _connectionFactory.Connect(_connection); - var toRemove = _entityIdentity.Id != null ? - new[] { await connection.Workspaces.FindAsync(_entityIdentity.Id) } : + var toRemove = _entityIdentity.Id != null ? [await connection.Workspaces.FindAsync(_entityIdentity.Id)] + : (await connection.Workspaces.ListAsync(ownerId: _entityOwner.OwnerId, shared: _entityOwner.IncludeShared)) .Where(workspace => _entityIdentity.Title == workspace.Title) .ToArray(); diff --git a/src/SeqCli/Cli/Options.cs b/src/SeqCli/Cli/Options.cs index 5678ac29..94ad1911 100644 --- a/src/SeqCli/Cli/Options.cs +++ b/src/SeqCli/Cli/Options.cs @@ -239,7 +239,7 @@ private static int GetLineEnd (int start, int length, string description) class OptionValueCollection : IList, IList { - List values = new List (); + List values = []; OptionContext c; internal OptionValueCollection (OptionContext c) @@ -313,7 +313,7 @@ public string this [int index] { public List ToList () { - return new List (values); + return [..values]; } public string[] ToArray () @@ -403,7 +403,7 @@ protected Option (string prototype, string description, int maxValueCount, bool this.names = (this is OptionSet.Category) // append GetHashCode() so that "duplicate" categories have distinct // names, e.g. adding multiple "" categories should be valid. - ? new[]{prototype + this.GetHashCode ()} + ? [prototype + this.GetHashCode ()] : prototype.Split ('|'); if (this is OptionSet.Category) @@ -472,12 +472,12 @@ protected static T Parse (string value, OptionContext c) internal string[] Names {get {return names;}} internal string[] ValueSeparators {get {return separators;}} - static readonly char[] NameTerminator = new char[]{'=', ':'}; + static readonly char[] NameTerminator = ['=', ':']; private OptionValueType ParsePrototype () { char type = '\0'; - List seps = new List (); + List seps = []; for (int i = 0; i < names.Length; ++i) { string name = names [i]; if (name.Length == 0) @@ -505,7 +505,7 @@ private OptionValueType ParsePrototype () "prototype"); if (count > 1) { if (seps.Count == 0) - this.separators = new string[]{":", "="}; + this.separators = [":", "="]; else if (seps.Count == 1 && seps [0].Length == 0) this.separators = null; else @@ -631,7 +631,7 @@ class ResponseFileSource : ArgumentSource { public override string[] GetNames () { - return new string[]{"@file"}; + return ["@file"]; } public override string Description { @@ -694,7 +694,7 @@ public Converter MessageLocalizer { get {return localizer;} } - List sources = new List (); + List sources = []; ReadOnlyCollection roSources; public ReadOnlyCollection ArgumentSources { @@ -936,7 +936,7 @@ public List Parse (IEnumerable arguments) OptionContext c = CreateOptionContext (); c.OptionIndex = -1; bool process = true; - List unprocessed = new List (); + List unprocessed = []; Option def = Contains ("<>") ? this ["<>"] : null; ArgumentEnumerator ae = new ArgumentEnumerator (arguments); foreach (string argument in ae) { @@ -960,7 +960,7 @@ public List Parse (IEnumerable arguments) } class ArgumentEnumerator : IEnumerable { - List> sources = new List> (); + List> sources = []; public ArgumentEnumerator (IEnumerable arguments) { @@ -1080,7 +1080,7 @@ private void ParseValue (string option, OptionContext c) if (option != null) foreach (string o in c.Option.ValueSeparators != null ? option.Split (c.Option.ValueSeparators, c.Option.MaxValueCount - c.OptionValues.Count, StringSplitOptions.None) - : new string[]{option}) { + : [option]) { c.OptionValues.Add (o); } if (c.OptionValues.Count == c.Option.MaxValueCount || @@ -1298,9 +1298,9 @@ private static string GetArgumentName (int index, int maxIndex, string descripti return maxIndex == 1 ? "VALUE" : index == 0 ? "NAME" : "VALUE"; string[] nameStart; if (maxIndex == 1) - nameStart = new string[]{"{0:", "{"}; + nameStart = ["{0:", "{"]; else - nameStart = new string[]{"{" + index + ":"}; + nameStart = ["{" + index + ":"]; for (int i = 0; i < nameStart.Length; ++i) { int start, j = 0; do { diff --git a/src/SeqCli/Csv/CsvTokenizer.cs b/src/SeqCli/Csv/CsvTokenizer.cs index aca5796a..2f3529a4 100644 --- a/src/SeqCli/Csv/CsvTokenizer.cs +++ b/src/SeqCli/Csv/CsvTokenizer.cs @@ -42,7 +42,7 @@ protected override IEnumerable> Tokenize(TextSpan span) if (next.Value != '"') { - yield return Result.Empty(next.Location, new[] {"double-quote"}); + yield return Result.Empty(next.Location, ["double-quote"]); yield break; } @@ -79,13 +79,13 @@ protected override IEnumerable> Tokenize(TextSpan span) } else { - yield return Result.Empty(next.Location, new[] {"comma", "newline"}); + yield return Result.Empty(next.Location, ["comma", "newline"]); yield break; } } else { - yield return Result.Empty(next.Location, new[] {"double-quote"}); + yield return Result.Empty(next.Location, ["double-quote"]); yield break; } diff --git a/src/SeqCli/Forwarder/Schema/EventSchema.cs b/src/SeqCli/Forwarder/Schema/EventSchema.cs index 104a08d3..44dad50a 100644 --- a/src/SeqCli/Forwarder/Schema/EventSchema.cs +++ b/src/SeqCli/Forwarder/Schema/EventSchema.cs @@ -13,10 +13,7 @@ static class EventSchema { static readonly MessageTemplateParser MessageTemplateParser = new MessageTemplateParser(); - static readonly HashSet ClefReifiedProperties = new HashSet - { - "@t", "@m", "@mt", "@l", "@x", "@i", "@r" - }; + static readonly HashSet ClefReifiedProperties = ["@t", "@m", "@mt", "@l", "@x", "@i", "@r"]; public static bool FromClefFormat(in int lineNumber, JObject compactFormat, [MaybeNullWhen(false)] out JObject rawFormat, [MaybeNullWhen(true)] out string error) { diff --git a/src/SeqCli/PlainText/Framing/FrameReader.cs b/src/SeqCli/PlainText/Framing/FrameReader.cs index c39bdb0e..d0f769e0 100644 --- a/src/SeqCli/PlainText/Framing/FrameReader.cs +++ b/src/SeqCli/PlainText/Framing/FrameReader.cs @@ -58,7 +58,7 @@ public async Task TryReadAsync() } else if (_unawaitedNextLine != null) { - var index = Task.WaitAny(new Task[] {_unawaitedNextLine}, _trailingLineArrivalDeadline); + var index = Task.WaitAny([_unawaitedNextLine], _trailingLineArrivalDeadline); if (index == -1) return new Frame(); @@ -81,7 +81,7 @@ public async Task TryReadAsync() while (true) { readLine = readLine ?? Task.Run(_source.ReadLineAsync); - var index = Task.WaitAny(new Task[] {readLine}, _trailingLineArrivalDeadline); + var index = Task.WaitAny([readLine], _trailingLineArrivalDeadline); if (index == -1) { if (hasValue) diff --git a/src/SeqCli/PlainText/ReifiedProperties.cs b/src/SeqCli/PlainText/ReifiedProperties.cs index 98bc98f0..9b63c64d 100644 --- a/src/SeqCli/PlainText/ReifiedProperties.cs +++ b/src/SeqCli/PlainText/ReifiedProperties.cs @@ -13,10 +13,7 @@ public const string SpanId = "@sp", TraceId = "@tr"; - static readonly HashSet All = new() - { - Message, Timestamp, Level, Exception, StartTimestamp, SpanId, TraceId - }; + static readonly HashSet All = [Message, Timestamp, Level, Exception, StartTimestamp, SpanId, TraceId]; public static bool IsReifiedProperty(string name) { diff --git a/test/SeqCli.Tests/Cli/CommandLineHostTests.cs b/test/SeqCli.Tests/Cli/CommandLineHostTests.cs index 9d9ba88a..78bea1e3 100644 --- a/test/SeqCli.Tests/Cli/CommandLineHostTests.cs +++ b/test/SeqCli.Tests/Cli/CommandLineHostTests.cs @@ -26,7 +26,7 @@ public async Task CheckCommandLineHostPicksCorrectCommand() new CommandMetadata {Name = "test2"}) }; var commandLineHost = new CommandLineHost(availableCommands); - await commandLineHost.Run(new []{ "test"},new LoggingLevelSwitch()); + await commandLineHost.Run(["test"],new LoggingLevelSwitch()); Assert.Equal("test", executed.First()); } @@ -46,7 +46,7 @@ public async Task WhenMoreThanOneSubcommandAndTheUserRunsWithSubcommandEnsurePic new CommandMetadata {Name = "test", SubCommand = "subcommand2"}) }; var commandLineHost = new CommandLineHost(availableCommands); - await commandLineHost.Run(new[] { "test", "subcommand2" }, new LoggingLevelSwitch()); + await commandLineHost.Run(["test", "subcommand2"], new LoggingLevelSwitch()); Assert.Equal("test-subcommand2", commandsRan.First()); } @@ -66,7 +66,7 @@ public async Task VerboseOptionSetsLoggingLevelToInformation() var commandLineHost = new CommandLineHost(availableCommands); - await commandLineHost.Run(new[] { "test", "--verbose" }, levelSwitch); + await commandLineHost.Run(["test", "--verbose"], levelSwitch); Assert.Equal(LogEventLevel.Information, levelSwitch.MinimumLevel); } diff --git a/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs index 3466c41b..8231d82b 100644 --- a/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs +++ b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs @@ -58,8 +58,8 @@ public void EntriesSurviveReloads() using var tmp = new TempFolder("Buffer"); using (var map = CreateActiveLogBufferMap(tmp)) { - map.GetLogBuffer(null).Enqueue(new[] {value}); - map.GetLogBuffer(apiKey).Enqueue(new[] {value}); + map.GetLogBuffer(null).Enqueue([value]); + map.GetLogBuffer(apiKey).Enqueue([value]); } using (var map = CreateActiveLogBufferMap(tmp)) diff --git a/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs b/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs index 469ed4c9..9ad315ac 100644 --- a/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs +++ b/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs @@ -23,7 +23,7 @@ public void PeekingDoesNotChangeState() { using var temp = TempFolder.ForCaller(); using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - buffer.Enqueue(new[] { Some.Bytes(140) }); + buffer.Enqueue([Some.Bytes(140)]); var contents = buffer.Peek((int)DefaultBufferSize); Assert.Single(contents); @@ -38,8 +38,8 @@ public void EnqueuedEntriesAreDequeuedFifo() using var temp = TempFolder.ForCaller(); using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue(new[] { a1, a2 }); - buffer.Enqueue(new[] { a3 }); + buffer.Enqueue([a1, a2]); + buffer.Enqueue([a3]); var contents = buffer.Peek((int)DefaultBufferSize); @@ -55,7 +55,7 @@ public void EntriesOverLimitArePurgedFifo() using var temp = TempFolder.ForCaller(); using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), 4096); byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue(new[] { a1, a2, a3 }); + buffer.Enqueue([a1, a2, a3]); var contents = buffer.Peek((int)DefaultBufferSize); @@ -70,7 +70,7 @@ public void SizeHintLimitsDequeuedEventCount() using var temp = TempFolder.ForCaller(); using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue(new[] { a1, a2, a3 }); + buffer.Enqueue([a1, a2, a3]); var contents = buffer.Peek(300); @@ -85,7 +85,7 @@ public void AtLeastOneEventIsAlwaysDequeued() using var temp = TempFolder.ForCaller(); using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue(new[] { a1, a2, a3 }); + buffer.Enqueue([a1, a2, a3]); var contents = buffer.Peek(30); @@ -99,7 +99,7 @@ public void GivingTheLastSeenEventKeyRemovesPrecedingEvents() using var temp = TempFolder.ForCaller(); using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue(new[] { a1, a2, a3 }); + buffer.Enqueue([a1, a2, a3]); var contents = buffer.Peek(420); Assert.Equal(3, contents.Length); @@ -116,12 +116,12 @@ public void GivingTheLastSeeEventKeyLeavesSuccessiveEvents() using var temp = TempFolder.ForCaller(); using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue(new[] { a1, a2, a3 }); + buffer.Enqueue([a1, a2, a3]); var contents = buffer.Peek(30); Assert.Single(contents); - buffer.Enqueue(new [] { Some.Bytes(140) }); + buffer.Enqueue([Some.Bytes(140)]); buffer.Dequeue(contents[0].Key); @@ -135,7 +135,7 @@ public void EnumerationIsInOrder() using var temp = TempFolder.ForCaller(); using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue(new[] { a1, a2, a3 }); + buffer.Enqueue([a1, a2, a3]); var contents = new List(); buffer.Enumerate((k, v) => diff --git a/test/SeqCli.Tests/Signals/SignalExpressionParserTests.cs b/test/SeqCli.Tests/Signals/SignalExpressionParserTests.cs index 92eba39d..6b9e2fad 100644 --- a/test/SeqCli.Tests/Signals/SignalExpressionParserTests.cs +++ b/test/SeqCli.Tests/Signals/SignalExpressionParserTests.cs @@ -17,15 +17,15 @@ public void ParseSuccessfully((string, string) inputs) } public static IEnumerable _sources = new []{ - new object[] { ("signal-1 ", "signal-1") }, - - new object[] { ("(signal-1)", "signal-1") }, - - new object[] { ("signal-1 ,signal-2", "signal-1,signal-2") }, - - new object[] { (" signal-1,signal-2~ signal-3", "(signal-1,signal-2)~signal-3") }, - - new object[] { ("signal-1,signal-2,(signal-3~signal-4)", "(signal-1,signal-2),(signal-3~signal-4)") }, + [("signal-1 ", "signal-1")], + + [("(signal-1)", "signal-1")], + + [("signal-1 ,signal-2", "signal-1,signal-2")], + + [(" signal-1,signal-2~ signal-3", "(signal-1,signal-2)~signal-3")], + + [("signal-1,signal-2,(signal-3~signal-4)", "(signal-1,signal-2),(signal-3~signal-4)")], new object[] { ("signal-1~( (signal-2~signal-3) ,signal-4)", "signal-1~((signal-2~signal-3),signal-4)") } }; diff --git a/test/SeqCli.Tests/Templates/TemplateWriterTests.cs b/test/SeqCli.Tests/Templates/TemplateWriterTests.cs index 49629e07..84707b47 100644 --- a/test/SeqCli.Tests/Templates/TemplateWriterTests.cs +++ b/test/SeqCli.Tests/Templates/TemplateWriterTests.cs @@ -29,8 +29,8 @@ public async Task WritesTemplates() Id = "test-stuff", Name = "Test Stuff", ReferencedId = "test-ref", - Numbers = new List { 1, 2, 3 }, - Strings = new List { "test" }, + Numbers = [1, 2, 3], + Strings = ["test"], Dictionary = new Dictionary{ ["First"] = "a" } }; From a2221ba330222ff7e792651f6051d1e81a1398f3 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 29 Feb 2024 15:08:07 +1000 Subject: [PATCH 15/51] Clean up code style --- seqcli.sln.DotSettings | 3 + .../Cli/Commands/Forwarder/RunCommand.cs | 302 ++- .../Cli/Commands/Forwarder/TruncateCommand.cs | 39 +- src/SeqCli/Cli/Options.cs | 1998 ++++++++--------- .../Config/Forwarder/ForwarderApiConfig.cs | 2 + .../Config/Forwarder/ForwarderConfig.cs | 2 + .../Forwarder/ForwarderDiagnosticConfig.cs | 2 + .../Forwarder/ForwarderStorageConfig.cs | 2 + src/SeqCli/Config/OutputConfig.cs | 3 + src/SeqCli/Config/SeqCliConfig.cs | 9 +- .../Cryptography/IStringDataProtector.cs | 13 +- .../Cryptography/StringDataProtector.cs | 11 +- .../Cryptography/UnprotectedStringData.cs | 21 +- .../Forwarder/Diagnostics/InMemorySink.cs | 43 +- .../Forwarder/Diagnostics/IngestionLog.cs | 71 +- src/SeqCli/Forwarder/ForwarderModule.cs | 84 + .../Forwarder/Multiplexing/ActiveLogBuffer.cs | 31 +- .../Multiplexing/ActiveLogBufferMap.cs | 299 ++- .../Multiplexing/HttpLogShipperFactory.cs | 33 +- .../Multiplexing/ILogShipperFactory.cs | 11 +- .../Multiplexing/InertLogShipperFactory.cs | 13 +- .../Multiplexing/ServerResponseProxy.cs | 49 +- .../Forwarder/Properties/AssemblyInfo.cs | 3 - src/SeqCli/Forwarder/Schema/EventSchema.cs | 255 ++- src/SeqCli/Forwarder/SeqForwarderModule.cs | 85 - .../ExponentialBackoffConnectionSchedule.cs | 79 +- .../Forwarder/Shipper/HttpLogShipper.cs | 341 ++- .../Forwarder/Shipper/InertLogShipper.cs | 23 +- src/SeqCli/Forwarder/Shipper/LogShipper.cs | 15 +- src/SeqCli/Forwarder/Shipper/SeqApi.cs | 9 +- src/SeqCli/Forwarder/Storage/LogBuffer.cs | 429 ++-- .../Forwarder/Storage/LogBufferEntry.cs | 11 +- src/SeqCli/Forwarder/Util/CaptiveProcess.cs | 103 +- .../Forwarder/Util/EnumerableExtensions.cs | 21 +- .../Forwarder/Util/ExecutionEnvironment.cs | 19 +- .../Forwarder/Util/UnclosableStreamWrapper.cs | 77 +- .../Forwarder/Web/Api/ApiRootController.cs | 55 +- .../Forwarder/Web/Api/IngestionController.cs | 323 ++- .../Forwarder/Web/Host/ServerService.cs | 69 +- src/SeqCli/Forwarder/Web/Host/Startup.cs | 55 +- .../Web/RequestProcessingException.cs | 19 +- src/SeqCli/Syntax/QueryBuilder.cs | 8 +- test/SeqCli.EndToEnd/Args.cs | 2 +- .../SeqCli.EndToEnd/Support/CaptiveProcess.cs | 8 +- .../Multiplexing/ActiveLogBufferMapTests.cs | 125 +- .../Forwarder/Schema/EventSchemaTests.cs | 109 +- .../Shipper/ServerResponseProxyTests.cs | 73 +- .../Forwarder/Storage/LogBufferTests.cs | 237 +- .../PlainText/NameValueExtractorTests.cs | 2 +- test/SeqCli.Tests/Support/TempFolder.cs | 65 +- 50 files changed, 2817 insertions(+), 2844 deletions(-) create mode 100644 src/SeqCli/Forwarder/ForwarderModule.cs delete mode 100644 src/SeqCli/Forwarder/Properties/AssemblyInfo.cs delete mode 100644 src/SeqCli/Forwarder/SeqForwarderModule.cs diff --git a/seqcli.sln.DotSettings b/seqcli.sln.DotSettings index a31aab98..6c218398 100644 --- a/seqcli.sln.DotSettings +++ b/seqcli.sln.DotSettings @@ -1,5 +1,6 @@  IO + IP MS True True @@ -17,6 +18,7 @@ True True True + True True True True @@ -33,4 +35,5 @@ True True True + True True \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index ca81665b..e743f174 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -16,7 +16,6 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net; -using System.Security.Cryptography.X509Certificates; using System.Threading.Tasks; using Autofac; using Autofac.Extensions.DependencyInjection; @@ -35,76 +34,76 @@ // ReSharper disable UnusedType.Global -namespace SeqCli.Cli.Commands.Forwarder +namespace SeqCli.Cli.Commands.Forwarder; + +[Command("forwarder", "run", "Run the server interactively")] +class RunCommand : Command { - [Command("forwarder", "run", "Run the server interactively")] - class RunCommand : Command - { - readonly StoragePathFeature _storagePath; - readonly ListenUriFeature _listenUri; + readonly StoragePathFeature _storagePath; + readonly ListenUriFeature _listenUri; - bool _noLogo; + bool _noLogo; - public RunCommand() - { - Options.Add("nologo", _ => _noLogo = true); - _storagePath = Enable(); - _listenUri = Enable(); - } + public RunCommand() + { + Options.Add("nologo", _ => _noLogo = true); + _storagePath = Enable(); + _listenUri = Enable(); + } - protected override async Task Run(string[] unrecognized) + protected override async Task Run(string[] unrecognized) + { + if (Environment.UserInteractive) { - if (Environment.UserInteractive) + if (!_noLogo) { - if (!_noLogo) - { - WriteBanner(); - Console.WriteLine(); - } - - Console.WriteLine("Running as server; press Ctrl+C to exit."); + WriteBanner(); Console.WriteLine(); } - SeqCliConfig config; + Console.WriteLine("Running as server; press Ctrl+C to exit."); + Console.WriteLine(); + } - try - { - config = SeqCliConfig.Read(); // _storagePath.ConfigFilePath); - } - catch (Exception ex) - { - await using var logger = CreateLogger( - LogEventLevel.Information, - ForwarderDiagnosticConfig.GetDefaultInternalLogPath()); + SeqCliConfig config; - logger.Fatal(ex, "Failed to load configuration from {ConfigFilePath}", _storagePath.ConfigFilePath); - return 1; - } + try + { + config = SeqCliConfig.Read(); // _storagePath.ConfigFilePath); + } + catch (Exception ex) + { + await using var logger = CreateLogger( + LogEventLevel.Information, + ForwarderDiagnosticConfig.GetDefaultInternalLogPath()); - Log.Logger = CreateLogger( - config.Forwarder.Diagnostics.InternalLoggingLevel, - config.Forwarder.Diagnostics.InternalLogPath, - config.Forwarder.Diagnostics.InternalLogServerUri, - config.Forwarder.Diagnostics.InternalLogServerApiKey); + logger.Fatal(ex, "Failed to load configuration from {ConfigFilePath}", _storagePath.ConfigFilePath); + return 1; + } - var listenUri = _listenUri.ListenUri ?? config.Forwarder.Api.ListenUri; + Log.Logger = CreateLogger( + config.Forwarder.Diagnostics.InternalLoggingLevel, + config.Forwarder.Diagnostics.InternalLogPath, + config.Forwarder.Diagnostics.InternalLogServerUri, + config.Forwarder.Diagnostics.InternalLogServerApiKey); - try - { - ILifetimeScope? container = null; - using var host = new HostBuilder() - .UseSerilog() - .UseServiceProviderFactory(new AutofacServiceProviderFactory()) - .ConfigureContainer(builder => - { - builder.RegisterBuildCallback(ls => container = ls); - builder.RegisterModule(new SeqForwarderModule(_storagePath.BufferPath, config)); - }) - .ConfigureWebHostDefaults(web => - { - web.UseStartup(); - web.UseKestrel(options => + var listenUri = _listenUri.ListenUri ?? config.Forwarder.Api.ListenUri; + + try + { + ILifetimeScope? container = null; + using var host = new HostBuilder() + .UseSerilog() + .UseServiceProviderFactory(new AutofacServiceProviderFactory()) + .ConfigureContainer(builder => + { + builder.RegisterBuildCallback(ls => container = ls); + builder.RegisterModule(new ForwarderModule(_storagePath.BufferPath, config)); + }) + .ConfigureWebHostDefaults(web => + { + web.UseStartup(); + web.UseKestrel(options => { options.AddServerHeader = false; options.AllowSynchronousIO = true; @@ -139,124 +138,123 @@ protected override async Task Run(string[] unrecognized) options.Listen(ipAddress, apiListenUri.Port); } }); - }) - .Build(); + }) + .Build(); - if (container == null) throw new Exception("Host did not build container."); + if (container == null) throw new Exception("Host did not build container."); - var service = container.Resolve( - new TypedParameter(typeof(IHost), host), - new NamedParameter("listenUri", listenUri)); + var service = container.Resolve( + new TypedParameter(typeof(IHost), host), + new NamedParameter("listenUri", listenUri)); - var exit = ExecutionEnvironment.SupportsStandardIO - ? RunStandardIO(service, Console.Out) - : RunService(service); + var exit = ExecutionEnvironment.SupportsStandardIO + ? RunStandardIO(service, Console.Out) + : RunService(service); - return exit; - } - catch (Exception ex) - { - Log.Fatal(ex, "Unhandled exception"); - return -1; - } - finally - { - await Log.CloseAndFlushAsync(); - } + return exit; } - - static Logger CreateLogger( - LogEventLevel internalLoggingLevel, - string internalLogPath, - string? internalLogServerUri = null, - string? internalLogServerApiKey = null) + catch (Exception ex) { - var loggerConfiguration = new LoggerConfiguration() - .Enrich.FromLogContext() - .Enrich.WithProperty("MachineName", Environment.MachineName) - .Enrich.WithProperty("Application", "Seq Forwarder") - .MinimumLevel.Is(internalLoggingLevel) - .MinimumLevel.Override("Microsoft", LogEventLevel.Warning) - .WriteTo.File( - new RenderedCompactJsonFormatter(), - GetRollingLogFilePathFormat(internalLogPath), - rollingInterval: RollingInterval.Day, - fileSizeLimitBytes: 1024 * 1024); - - if (Environment.UserInteractive) - loggerConfiguration.WriteTo.Console(restrictedToMinimumLevel: LogEventLevel.Information); - - if (!string.IsNullOrWhiteSpace(internalLogServerUri)) - loggerConfiguration.WriteTo.Seq( - internalLogServerUri, - apiKey: internalLogServerApiKey); - - return loggerConfiguration.CreateLogger(); + Log.Fatal(ex, "Unhandled exception"); + return -1; } - - static string GetRollingLogFilePathFormat(string internalLogPath) + finally { - if (internalLogPath == null) throw new ArgumentNullException(nameof(internalLogPath)); - - return Path.Combine(internalLogPath, "seq-forwarder-.log"); + await Log.CloseAndFlushAsync(); } + } - [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] - static int RunService(ServerService service) - { + [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] + static int RunService(ServerService service) + { #if WINDOWS System.ServiceProcess.ServiceBase.Run([ new ServiceProcess.SeqForwarderWindowsService(service) ]); return 0; #else - throw new NotSupportedException("Windows services are not supported on this platform."); + throw new NotSupportedException("Windows services are not supported on this platform."); #endif - } + } - static int RunStandardIO(ServerService service, TextWriter cout) + static int RunStandardIO(ServerService service, TextWriter cout) + { + service.Start(); + + try { - service.Start(); + Console.TreatControlCAsInput = true; + var k = Console.ReadKey(true); + while (k.Key != ConsoleKey.C || !k.Modifiers.HasFlag(ConsoleModifiers.Control)) + k = Console.ReadKey(true); - try - { - Console.TreatControlCAsInput = true; - var k = Console.ReadKey(true); - while (k.Key != ConsoleKey.C || !k.Modifiers.HasFlag(ConsoleModifiers.Control)) - k = Console.ReadKey(true); + cout.WriteLine("Ctrl+C pressed; stopping..."); + Console.TreatControlCAsInput = false; + } + catch (Exception ex) + { + Log.Debug(ex, "Console not attached, waiting for any input"); + Console.Read(); + } - cout.WriteLine("Ctrl+C pressed; stopping..."); - Console.TreatControlCAsInput = false; - } - catch (Exception ex) - { - Log.Debug(ex, "Console not attached, waiting for any input"); - Console.Read(); - } + service.Stop(); - service.Stop(); + return 0; + } - return 0; - } + static void WriteBanner() + { + Write("─", ConsoleColor.DarkGray, 47); + Console.WriteLine(); + Write(" Seq Forwarder", ConsoleColor.White); + Write(" ──", ConsoleColor.DarkGray); + Write(" © 2024 Datalust Pty Ltd", ConsoleColor.Gray); + Console.WriteLine(); + Write("─", ConsoleColor.DarkGray, 47); + Console.WriteLine(); + } - static void WriteBanner() - { - Write("─", ConsoleColor.DarkGray, 47); - Console.WriteLine(); - Write(" Seq Forwarder", ConsoleColor.White); - Write(" ──", ConsoleColor.DarkGray); - Write(" © 2024 Datalust Pty Ltd", ConsoleColor.Gray); - Console.WriteLine(); - Write("─", ConsoleColor.DarkGray, 47); - Console.WriteLine(); - } + static void Write(string s, ConsoleColor color, int repeats = 1) + { + Console.ForegroundColor = color; + for (var i = 0; i < repeats; ++i) + Console.Write(s); + Console.ResetColor(); + } + + static Logger CreateLogger( + LogEventLevel internalLoggingLevel, + string internalLogPath, + string? internalLogServerUri = null, + string? internalLogServerApiKey = null) + { + var loggerConfiguration = new LoggerConfiguration() + .Enrich.FromLogContext() + .Enrich.WithProperty("MachineName", Environment.MachineName) + .Enrich.WithProperty("Application", "Seq Forwarder") + .MinimumLevel.Is(internalLoggingLevel) + .MinimumLevel.Override("Microsoft", LogEventLevel.Warning) + .WriteTo.File( + new RenderedCompactJsonFormatter(), + GetRollingLogFilePathFormat(internalLogPath), + rollingInterval: RollingInterval.Day, + fileSizeLimitBytes: 1024 * 1024); - static void Write(string s, ConsoleColor color, int repeats = 1) - { - Console.ForegroundColor = color; - for (var i = 0; i < repeats; ++i) - Console.Write(s); - Console.ResetColor(); - } + if (Environment.UserInteractive) + loggerConfiguration.WriteTo.Console(restrictedToMinimumLevel: LogEventLevel.Information); + + if (!string.IsNullOrWhiteSpace(internalLogServerUri)) + loggerConfiguration.WriteTo.Seq( + internalLogServerUri, + apiKey: internalLogServerApiKey); + + return loggerConfiguration.CreateLogger(); + } + + static string GetRollingLogFilePathFormat(string internalLogPath) + { + if (internalLogPath == null) throw new ArgumentNullException(nameof(internalLogPath)); + + return Path.Combine(internalLogPath, "seq-forwarder-.log"); } -} +} \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs index 856c8f4d..df5130da 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs @@ -18,32 +18,31 @@ using SeqCli.Forwarder.Multiplexing; using Serilog; -namespace SeqCli.Cli.Commands.Forwarder +namespace SeqCli.Cli.Commands.Forwarder; + +[Command("forwarder", "truncate", "Clear the log buffer contents")] +class TruncateCommand : Command { - [Command("forwarder", "truncate", "Clear the log buffer contents")] - class TruncateCommand : Command + readonly StoragePathFeature _storagePath; + + public TruncateCommand() { - readonly StoragePathFeature _storagePath; + _storagePath = Enable(); + } - public TruncateCommand() + protected override async Task Run(string[] args) + { + try { - _storagePath = Enable(); + ActiveLogBufferMap.Truncate(_storagePath.BufferPath); + return 0; } - - protected override async Task Run(string[] args) + catch (Exception ex) { - try - { - ActiveLogBufferMap.Truncate(_storagePath.BufferPath); - return 0; - } - catch (Exception ex) - { - await using var logger = new LoggerConfiguration().WriteTo.Console().CreateLogger(); + await using var logger = new LoggerConfiguration().WriteTo.Console().CreateLogger(); - logger.Fatal(ex, "Could not truncate log buffer"); - return 1; - } + logger.Fatal(ex, "Could not truncate log buffer"); + return 1; } } -} +} \ No newline at end of file diff --git a/src/SeqCli/Cli/Options.cs b/src/SeqCli/Cli/Options.cs index 94ad1911..beedaad4 100644 --- a/src/SeqCli/Cli/Options.cs +++ b/src/SeqCli/Cli/Options.cs @@ -150,1218 +150,1216 @@ #if NDESK_OPTIONS namespace NDesk.Options #else -namespace SeqCli.Cli +namespace SeqCli.Cli; #endif -{ - delegate U Converter(T t); - static class StringCoda { +delegate U Converter(T t); - public static IEnumerable WrappedLines (string self, params int[] widths) - { - IEnumerable w = widths; - return WrappedLines (self, w); - } +static class StringCoda { - public static IEnumerable WrappedLines (string self, IEnumerable widths) - { - if (widths == null) - throw new ArgumentNullException (nameof(widths)); - return CreateWrappedLinesIterator (self, widths); - } + public static IEnumerable WrappedLines (string self, params int[] widths) + { + IEnumerable w = widths; + return WrappedLines (self, w); + } - private static IEnumerable CreateWrappedLinesIterator (string self, IEnumerable widths) - { - if (string.IsNullOrEmpty (self)) { - yield return string.Empty; - yield break; - } - using (IEnumerator ewidths = widths.GetEnumerator ()) { - bool? hw = null; - int width = GetNextWidth (ewidths, int.MaxValue, ref hw); - int start = 0, end; - do { - end = GetLineEnd (start, width, self); - char c = self [end-1]; - if (char.IsWhiteSpace (c)) - --end; - bool needContinuation = end != self.Length && !IsEolChar (c); - string continuation = ""; - if (needContinuation) { - --end; - continuation = "-"; - } - string line = self.Substring (start, end - start) + continuation; - yield return line; - start = end; - if (char.IsWhiteSpace (c)) - ++start; - width = GetNextWidth (ewidths, width, ref hw); - } while (start < self.Length); - } - } + public static IEnumerable WrappedLines (string self, IEnumerable widths) + { + if (widths == null) + throw new ArgumentNullException (nameof(widths)); + return CreateWrappedLinesIterator (self, widths); + } - private static int GetNextWidth (IEnumerator ewidths, int curWidth, ref bool? eValid) - { - if (!eValid.HasValue || (eValid.HasValue && eValid.Value)) { - curWidth = (eValid = ewidths.MoveNext ()).Value ? ewidths.Current : curWidth; - // '.' is any character, - is for a continuation - const string minWidth = ".-"; - if (curWidth < minWidth.Length) - throw new ArgumentOutOfRangeException ("widths", - string.Format ("Element must be >= {0}, was {1}.", minWidth.Length, curWidth)); - return curWidth; - } - // no more elements, use the last element. - return curWidth; + private static IEnumerable CreateWrappedLinesIterator (string self, IEnumerable widths) + { + if (string.IsNullOrEmpty (self)) { + yield return string.Empty; + yield break; + } + using (IEnumerator ewidths = widths.GetEnumerator ()) { + bool? hw = null; + int width = GetNextWidth (ewidths, int.MaxValue, ref hw); + int start = 0, end; + do { + end = GetLineEnd (start, width, self); + char c = self [end-1]; + if (char.IsWhiteSpace (c)) + --end; + bool needContinuation = end != self.Length && !IsEolChar (c); + string continuation = ""; + if (needContinuation) { + --end; + continuation = "-"; + } + string line = self.Substring (start, end - start) + continuation; + yield return line; + start = end; + if (char.IsWhiteSpace (c)) + ++start; + width = GetNextWidth (ewidths, width, ref hw); + } while (start < self.Length); } + } - private static bool IsEolChar (char c) - { - return !char.IsLetterOrDigit (c); + private static int GetNextWidth (IEnumerator ewidths, int curWidth, ref bool? eValid) + { + if (!eValid.HasValue || (eValid.HasValue && eValid.Value)) { + curWidth = (eValid = ewidths.MoveNext ()).Value ? ewidths.Current : curWidth; + // '.' is any character, - is for a continuation + const string minWidth = ".-"; + if (curWidth < minWidth.Length) + throw new ArgumentOutOfRangeException ("widths", + string.Format ("Element must be >= {0}, was {1}.", minWidth.Length, curWidth)); + return curWidth; } + // no more elements, use the last element. + return curWidth; + } - private static int GetLineEnd (int start, int length, string description) - { - int end = System.Math.Min (start + length, description.Length); - int sep = -1; - for (int i = start; i < end; ++i) { - if (description [i] == '\n') - return i+1; - if (IsEolChar (description [i])) - sep = i+1; - } - if (sep == -1 || end == description.Length) - return end; - return sep; - } + private static bool IsEolChar (char c) + { + return !char.IsLetterOrDigit (c); } - class OptionValueCollection : IList, IList { + private static int GetLineEnd (int start, int length, string description) + { + int end = System.Math.Min (start + length, description.Length); + int sep = -1; + for (int i = start; i < end; ++i) { + if (description [i] == '\n') + return i+1; + if (IsEolChar (description [i])) + sep = i+1; + } + if (sep == -1 || end == description.Length) + return end; + return sep; + } +} - List values = []; - OptionContext c; +class OptionValueCollection : IList, IList { - internal OptionValueCollection (OptionContext c) - { - this.c = c; - } + List values = []; + OptionContext c; - #region ICollection - void ICollection.CopyTo (Array array, int index) {(values as ICollection).CopyTo (array, index);} - bool ICollection.IsSynchronized {get {return (values as ICollection).IsSynchronized;}} - object ICollection.SyncRoot {get {return (values as ICollection).SyncRoot;}} - #endregion - - #region ICollection - public void Add (string item) {values.Add (item);} - public void Clear () {values.Clear ();} - public bool Contains (string item) {return values.Contains (item);} - public void CopyTo (string[] array, int arrayIndex) {values.CopyTo (array, arrayIndex);} - public bool Remove (string item) {return values.Remove (item);} - public int Count {get {return values.Count;}} - public bool IsReadOnly {get {return false;}} - #endregion - - #region IEnumerable - IEnumerator IEnumerable.GetEnumerator () {return values.GetEnumerator ();} - #endregion - - #region IEnumerable - public IEnumerator GetEnumerator () {return values.GetEnumerator ();} - #endregion - - #region IList - int IList.Add (object value) {return (values as IList).Add (value);} - bool IList.Contains (object value) {return (values as IList).Contains (value);} - int IList.IndexOf (object value) {return (values as IList).IndexOf (value);} - void IList.Insert (int index, object value) {(values as IList).Insert (index, value);} - void IList.Remove (object value) {(values as IList).Remove (value);} - void IList.RemoveAt (int index) {(values as IList).RemoveAt (index);} - bool IList.IsFixedSize {get {return false;}} - object IList.this [int index] {get {return this [index];} set {(values as IList)[index] = value;}} - #endregion - - #region IList - public int IndexOf (string item) {return values.IndexOf (item);} - public void Insert (int index, string item) {values.Insert (index, item);} - public void RemoveAt (int index) {values.RemoveAt (index);} - - private void AssertValid (int index) - { - if (c.Option == null) - throw new InvalidOperationException ("OptionContext.Option is null."); - if (index >= c.Option.MaxValueCount) - throw new ArgumentOutOfRangeException (nameof(index)); - if (c.Option.OptionValueType == OptionValueType.Required && - index >= values.Count) - throw new OptionException (string.Format ( - c.OptionSet.MessageLocalizer ("Missing required value for option '{0}'."), c.OptionName), - c.OptionName); - } + internal OptionValueCollection (OptionContext c) + { + this.c = c; + } - public string this [int index] { - get { - AssertValid (index); - return index >= values.Count ? null : values [index]; - } - set { - values [index] = value; - } - } - #endregion + #region ICollection + void ICollection.CopyTo (Array array, int index) {(values as ICollection).CopyTo (array, index);} + bool ICollection.IsSynchronized {get {return (values as ICollection).IsSynchronized;}} + object ICollection.SyncRoot {get {return (values as ICollection).SyncRoot;}} + #endregion + + #region ICollection + public void Add (string item) {values.Add (item);} + public void Clear () {values.Clear ();} + public bool Contains (string item) {return values.Contains (item);} + public void CopyTo (string[] array, int arrayIndex) {values.CopyTo (array, arrayIndex);} + public bool Remove (string item) {return values.Remove (item);} + public int Count {get {return values.Count;}} + public bool IsReadOnly {get {return false;}} + #endregion + + #region IEnumerable + IEnumerator IEnumerable.GetEnumerator () {return values.GetEnumerator ();} + #endregion + + #region IEnumerable + public IEnumerator GetEnumerator () {return values.GetEnumerator ();} + #endregion + + #region IList + int IList.Add (object value) {return (values as IList).Add (value);} + bool IList.Contains (object value) {return (values as IList).Contains (value);} + int IList.IndexOf (object value) {return (values as IList).IndexOf (value);} + void IList.Insert (int index, object value) {(values as IList).Insert (index, value);} + void IList.Remove (object value) {(values as IList).Remove (value);} + void IList.RemoveAt (int index) {(values as IList).RemoveAt (index);} + bool IList.IsFixedSize {get {return false;}} + object IList.this [int index] {get {return this [index];} set {(values as IList)[index] = value;}} + #endregion + + #region IList + public int IndexOf (string item) {return values.IndexOf (item);} + public void Insert (int index, string item) {values.Insert (index, item);} + public void RemoveAt (int index) {values.RemoveAt (index);} + + private void AssertValid (int index) + { + if (c.Option == null) + throw new InvalidOperationException ("OptionContext.Option is null."); + if (index >= c.Option.MaxValueCount) + throw new ArgumentOutOfRangeException (nameof(index)); + if (c.Option.OptionValueType == OptionValueType.Required && + index >= values.Count) + throw new OptionException (string.Format ( + c.OptionSet.MessageLocalizer ("Missing required value for option '{0}'."), c.OptionName), + c.OptionName); + } - public List ToList () - { - return [..values]; + public string this [int index] { + get { + AssertValid (index); + return index >= values.Count ? null : values [index]; } - - public string[] ToArray () - { - return values.ToArray (); + set { + values [index] = value; } + } + #endregion - public override string ToString () - { - return string.Join (", ", values.ToArray ()); - } + public List ToList () + { + return [..values]; } - class OptionContext { - private Option option; - private string name; - private int index; - private OptionSet set; - private OptionValueCollection c; + public string[] ToArray () + { + return values.ToArray (); + } - public OptionContext (OptionSet set) - { - this.set = set; - this.c = new OptionValueCollection (this); - } + public override string ToString () + { + return string.Join (", ", values.ToArray ()); + } +} - public Option Option { - get {return option;} - set {option = value;} - } +class OptionContext { + private Option option; + private string name; + private int index; + private OptionSet set; + private OptionValueCollection c; - public string OptionName { - get {return name;} - set {name = value;} - } + public OptionContext (OptionSet set) + { + this.set = set; + this.c = new OptionValueCollection (this); + } - public int OptionIndex { - get {return index;} - set {index = value;} - } + public Option Option { + get {return option;} + set {option = value;} + } - public OptionSet OptionSet { - get {return set;} - } + public string OptionName { + get {return name;} + set {name = value;} + } - public OptionValueCollection OptionValues { - get {return c;} - } + public int OptionIndex { + get {return index;} + set {index = value;} } - enum OptionValueType { - None, - Optional, - Required, + public OptionSet OptionSet { + get {return set;} } - abstract class Option { - string prototype, description; - string[] names; - OptionValueType type; - int count; - string[] separators; - bool hidden; + public OptionValueCollection OptionValues { + get {return c;} + } +} - protected Option (string prototype, string description) - : this (prototype, description, 1, false) - { - } +enum OptionValueType { + None, + Optional, + Required, +} - protected Option (string prototype, string description, int maxValueCount) - : this (prototype, description, maxValueCount, false) - { - } +abstract class Option { + string prototype, description; + string[] names; + OptionValueType type; + int count; + string[] separators; + bool hidden; - protected Option (string prototype, string description, int maxValueCount, bool hidden) - { - if (prototype == null) - throw new ArgumentNullException (nameof(prototype)); - if (prototype.Length == 0) - throw new ArgumentException ("Cannot be the empty string.", nameof(prototype)); - if (maxValueCount < 0) - throw new ArgumentOutOfRangeException (nameof(maxValueCount)); - - this.prototype = prototype; - this.description = description; - this.count = maxValueCount; - this.names = (this is OptionSet.Category) - // append GetHashCode() so that "duplicate" categories have distinct - // names, e.g. adding multiple "" categories should be valid. - ? [prototype + this.GetHashCode ()] - : prototype.Split ('|'); - - if (this is OptionSet.Category) - return; - - this.type = ParsePrototype (); - this.hidden = hidden; - - if (this.count == 0 && type != OptionValueType.None) - throw new ArgumentException ( - "Cannot provide maxValueCount of 0 for OptionValueType.Required or " + - "OptionValueType.Optional.", - nameof(maxValueCount)); - if (this.type == OptionValueType.None && maxValueCount > 1) - throw new ArgumentException ( - string.Format ("Cannot provide maxValueCount of {0} for OptionValueType.None.", maxValueCount), - nameof(maxValueCount)); - if (Array.IndexOf (names, "<>") >= 0 && - ((names.Length == 1 && this.type != OptionValueType.None) || - (names.Length > 1 && this.MaxValueCount > 1))) - throw new ArgumentException ( - "The default option handler '<>' cannot require values.", - nameof(prototype)); - } + protected Option (string prototype, string description) + : this (prototype, description, 1, false) + { + } - public string Prototype {get {return prototype;}} - public string Description {get {return description;}} - public OptionValueType OptionValueType {get {return type;}} - public int MaxValueCount {get {return count;}} - public bool Hidden {get {return hidden;}} + protected Option (string prototype, string description, int maxValueCount) + : this (prototype, description, maxValueCount, false) + { + } - public string[] GetNames () - { - return (string[]) names.Clone (); - } + protected Option (string prototype, string description, int maxValueCount, bool hidden) + { + if (prototype == null) + throw new ArgumentNullException (nameof(prototype)); + if (prototype.Length == 0) + throw new ArgumentException ("Cannot be the empty string.", nameof(prototype)); + if (maxValueCount < 0) + throw new ArgumentOutOfRangeException (nameof(maxValueCount)); + + this.prototype = prototype; + this.description = description; + this.count = maxValueCount; + this.names = (this is OptionSet.Category) + // append GetHashCode() so that "duplicate" categories have distinct + // names, e.g. adding multiple "" categories should be valid. + ? [prototype + this.GetHashCode ()] + : prototype.Split ('|'); + + if (this is OptionSet.Category) + return; + + this.type = ParsePrototype (); + this.hidden = hidden; + + if (this.count == 0 && type != OptionValueType.None) + throw new ArgumentException ( + "Cannot provide maxValueCount of 0 for OptionValueType.Required or " + + "OptionValueType.Optional.", + nameof(maxValueCount)); + if (this.type == OptionValueType.None && maxValueCount > 1) + throw new ArgumentException ( + string.Format ("Cannot provide maxValueCount of {0} for OptionValueType.None.", maxValueCount), + nameof(maxValueCount)); + if (Array.IndexOf (names, "<>") >= 0 && + ((names.Length == 1 && this.type != OptionValueType.None) || + (names.Length > 1 && this.MaxValueCount > 1))) + throw new ArgumentException ( + "The default option handler '<>' cannot require values.", + nameof(prototype)); + } - public string[] GetValueSeparators () - { - if (separators == null) - return new string [0]; - return (string[]) separators.Clone (); - } + public string Prototype {get {return prototype;}} + public string Description {get {return description;}} + public OptionValueType OptionValueType {get {return type;}} + public int MaxValueCount {get {return count;}} + public bool Hidden {get {return hidden;}} - protected static T Parse (string value, OptionContext c) - { - var tt = typeof (T).GetTypeInfo(); - bool nullable = tt.IsValueType && tt.IsGenericType && - !tt.IsGenericTypeDefinition && - tt.GetGenericTypeDefinition () == typeof (Nullable<>); - Type targetType = nullable ? tt.GetGenericArguments () [0] : typeof (T); - T t = default (T); - try { - if (value != null) - t = (T) Convert.ChangeType(value, targetType, CultureInfo.InvariantCulture); - } - catch (Exception e) { - throw new OptionException ( - string.Format ( - c.OptionSet.MessageLocalizer ("Could not convert string `{0}' to type {1} for option `{2}'."), - value, targetType.Name, c.OptionName), - c.OptionName, e); - } - return t; - } + public string[] GetNames () + { + return (string[]) names.Clone (); + } - internal string[] Names {get {return names;}} - internal string[] ValueSeparators {get {return separators;}} + public string[] GetValueSeparators () + { + if (separators == null) + return new string [0]; + return (string[]) separators.Clone (); + } - static readonly char[] NameTerminator = ['=', ':']; + protected static T Parse (string value, OptionContext c) + { + var tt = typeof (T).GetTypeInfo(); + bool nullable = tt.IsValueType && tt.IsGenericType && + !tt.IsGenericTypeDefinition && + tt.GetGenericTypeDefinition () == typeof (Nullable<>); + Type targetType = nullable ? tt.GetGenericArguments () [0] : typeof (T); + T t = default (T); + try { + if (value != null) + t = (T) Convert.ChangeType(value, targetType, CultureInfo.InvariantCulture); + } + catch (Exception e) { + throw new OptionException ( + string.Format ( + c.OptionSet.MessageLocalizer ("Could not convert string `{0}' to type {1} for option `{2}'."), + value, targetType.Name, c.OptionName), + c.OptionName, e); + } + return t; + } - private OptionValueType ParsePrototype () - { - char type = '\0'; - List seps = []; - for (int i = 0; i < names.Length; ++i) { - string name = names [i]; - if (name.Length == 0) - throw new ArgumentException ("Empty option names are not supported.", "prototype"); - - int end = name.IndexOfAny (NameTerminator); - if (end == -1) - continue; - names [i] = name.Substring (0, end); - if (type == '\0' || type == name [end]) - type = name [end]; - else - throw new ArgumentException ( - string.Format ("Conflicting option types: '{0}' vs. '{1}'.", type, name [end]), - "prototype"); - AddSeparators (name, end, seps); - } + internal string[] Names {get {return names;}} + internal string[] ValueSeparators {get {return separators;}} - if (type == '\0') - return OptionValueType.None; + static readonly char[] NameTerminator = ['=', ':']; - if (count <= 1 && seps.Count != 0) + private OptionValueType ParsePrototype () + { + char type = '\0'; + List seps = []; + for (int i = 0; i < names.Length; ++i) { + string name = names [i]; + if (name.Length == 0) + throw new ArgumentException ("Empty option names are not supported.", "prototype"); + + int end = name.IndexOfAny (NameTerminator); + if (end == -1) + continue; + names [i] = name.Substring (0, end); + if (type == '\0' || type == name [end]) + type = name [end]; + else throw new ArgumentException ( - string.Format ("Cannot provide key/value separators for Options taking {0} value(s).", count), - "prototype"); - if (count > 1) { - if (seps.Count == 0) - this.separators = [":", "="]; - else if (seps.Count == 1 && seps [0].Length == 0) - this.separators = null; - else - this.separators = seps.ToArray (); - } - - return type == '=' ? OptionValueType.Required : OptionValueType.Optional; + string.Format ("Conflicting option types: '{0}' vs. '{1}'.", type, name [end]), + "prototype"); + AddSeparators (name, end, seps); + } + + if (type == '\0') + return OptionValueType.None; + + if (count <= 1 && seps.Count != 0) + throw new ArgumentException ( + string.Format ("Cannot provide key/value separators for Options taking {0} value(s).", count), + "prototype"); + if (count > 1) { + if (seps.Count == 0) + this.separators = [":", "="]; + else if (seps.Count == 1 && seps [0].Length == 0) + this.separators = null; + else + this.separators = seps.ToArray (); } - private static void AddSeparators (string name, int end, ICollection seps) - { - int start = -1; - for (int i = end+1; i < name.Length; ++i) { - switch (name [i]) { - case '{': - if (start != -1) - throw new ArgumentException ( - string.Format ("Ill-formed name/value separator found in \"{0}\".", name), - "prototype"); - start = i+1; - break; - case '}': - if (start == -1) - throw new ArgumentException ( - string.Format ("Ill-formed name/value separator found in \"{0}\".", name), - "prototype"); - seps.Add (name.Substring (start, i-start)); - start = -1; - break; - default: - if (start == -1) - seps.Add (name [i].ToString ()); - break; - } + return type == '=' ? OptionValueType.Required : OptionValueType.Optional; + } + + private static void AddSeparators (string name, int end, ICollection seps) + { + int start = -1; + for (int i = end+1; i < name.Length; ++i) { + switch (name [i]) { + case '{': + if (start != -1) + throw new ArgumentException ( + string.Format ("Ill-formed name/value separator found in \"{0}\".", name), + "prototype"); + start = i+1; + break; + case '}': + if (start == -1) + throw new ArgumentException ( + string.Format ("Ill-formed name/value separator found in \"{0}\".", name), + "prototype"); + seps.Add (name.Substring (start, i-start)); + start = -1; + break; + default: + if (start == -1) + seps.Add (name [i].ToString ()); + break; } - if (start != -1) - throw new ArgumentException ( - string.Format ("Ill-formed name/value separator found in \"{0}\".", name), - "prototype"); } + if (start != -1) + throw new ArgumentException ( + string.Format ("Ill-formed name/value separator found in \"{0}\".", name), + "prototype"); + } - public void Invoke (OptionContext c) - { - OnParseComplete (c); - c.OptionName = null; - c.Option = null; - c.OptionValues.Clear (); - } + public void Invoke (OptionContext c) + { + OnParseComplete (c); + c.OptionName = null; + c.Option = null; + c.OptionValues.Clear (); + } - protected abstract void OnParseComplete (OptionContext c); + protected abstract void OnParseComplete (OptionContext c); - public override string ToString () - { - return Prototype; - } + public override string ToString () + { + return Prototype; } +} - abstract class ArgumentSource { +abstract class ArgumentSource { - protected ArgumentSource () - { - } + protected ArgumentSource () + { + } - public abstract string[] GetNames (); - public abstract string Description { get; } - public abstract bool GetArguments (string value, out IEnumerable replacement); + public abstract string[] GetNames (); + public abstract string Description { get; } + public abstract bool GetArguments (string value, out IEnumerable replacement); - public static IEnumerable GetArgumentsFromFile (string file) - { - return GetArguments (File.OpenText (file), true); - } + public static IEnumerable GetArgumentsFromFile (string file) + { + return GetArguments (File.OpenText (file), true); + } - public static IEnumerable GetArguments (TextReader reader) - { - return GetArguments (reader, false); - } + public static IEnumerable GetArguments (TextReader reader) + { + return GetArguments (reader, false); + } - // Cribbed from mcs/driver.cs:LoadArgs(string) - static IEnumerable GetArguments (TextReader reader, bool close) - { - try { - StringBuilder arg = new StringBuilder (); + // Cribbed from mcs/driver.cs:LoadArgs(string) + static IEnumerable GetArguments (TextReader reader, bool close) + { + try { + StringBuilder arg = new StringBuilder (); - string line; - while ((line = reader.ReadLine ()) != null) { - int t = line.Length; + string line; + while ((line = reader.ReadLine ()) != null) { + int t = line.Length; - for (int i = 0; i < t; i++) { - char c = line [i]; + for (int i = 0; i < t; i++) { + char c = line [i]; - if (c == '"' || c == '\'') { - char end = c; + if (c == '"' || c == '\'') { + char end = c; - for (i++; i < t; i++){ - c = line [i]; - - if (c == end) - break; - arg.Append (c); - } - } else if (c == ' ') { - if (arg.Length > 0) { - yield return arg.ToString (); - arg.Length = 0; - } - } else + for (i++; i < t; i++){ + c = line [i]; + + if (c == end) + break; arg.Append (c); - } - if (arg.Length > 0) { - yield return arg.ToString (); - arg.Length = 0; - } + } + } else if (c == ' ') { + if (arg.Length > 0) { + yield return arg.ToString (); + arg.Length = 0; + } + } else + arg.Append (c); + } + if (arg.Length > 0) { + yield return arg.ToString (); + arg.Length = 0; } } - finally { - if (close) - reader.Dispose(); - } + } + finally { + if (close) + reader.Dispose(); } } +} - class ResponseFileSource : ArgumentSource { +class ResponseFileSource : ArgumentSource { - public override string[] GetNames () - { - return ["@file"]; - } + public override string[] GetNames () + { + return ["@file"]; + } - public override string Description { - get {return "Read response file for more options.";} - } + public override string Description { + get {return "Read response file for more options.";} + } - public override bool GetArguments (string value, out IEnumerable replacement) - { - if (string.IsNullOrEmpty (value) || !value.StartsWith ("@")) { - replacement = null; - return false; - } - replacement = ArgumentSource.GetArgumentsFromFile (value.Substring (1)); - return true; + public override bool GetArguments (string value, out IEnumerable replacement) + { + if (string.IsNullOrEmpty (value) || !value.StartsWith ("@")) { + replacement = null; + return false; } + replacement = ArgumentSource.GetArgumentsFromFile (value.Substring (1)); + return true; } +} - class OptionException : Exception { - private string option; +class OptionException : Exception { + private string option; - public OptionException () - { - } + public OptionException () + { + } - public OptionException (string message, string optionName) - : base (message) - { - this.option = optionName; - } + public OptionException (string message, string optionName) + : base (message) + { + this.option = optionName; + } - public OptionException (string message, string optionName, Exception innerException) - : base (message, innerException) - { - this.option = optionName; - } + public OptionException (string message, string optionName, Exception innerException) + : base (message, innerException) + { + this.option = optionName; + } - public string OptionName { - get {return this.option;} - } + public string OptionName { + get {return this.option;} } +} - delegate void OptionAction (TKey key, TValue value); +delegate void OptionAction (TKey key, TValue value); - class OptionSet : KeyedCollection +class OptionSet : KeyedCollection +{ + public OptionSet () + : this (delegate (string f) {return f;}) { - public OptionSet () - : this (delegate (string f) {return f;}) - { - } + } - public OptionSet (Converter localizer) - { - this.localizer = localizer; - this.roSources = new ReadOnlyCollection(sources); - } + public OptionSet (Converter localizer) + { + this.localizer = localizer; + this.roSources = new ReadOnlyCollection(sources); + } - Converter localizer; + Converter localizer; - public Converter MessageLocalizer { - get {return localizer;} - } + public Converter MessageLocalizer { + get {return localizer;} + } + + List sources = []; + ReadOnlyCollection roSources; - List sources = []; - ReadOnlyCollection roSources; + public ReadOnlyCollection ArgumentSources { + get {return roSources;} + } - public ReadOnlyCollection ArgumentSources { - get {return roSources;} + + protected override string GetKeyForItem (Option item) + { + if (item == null) + throw new ArgumentNullException ("option"); + if (item.Names != null && item.Names.Length > 0) + return item.Names [0]; + // This should never happen, as it's invalid for Option to be + // constructed w/o any names. + throw new InvalidOperationException ("Option has no names!"); + } + + [Obsolete ("Use KeyedCollection.this[string]")] + protected Option GetOptionForName (string option) + { + if (option == null) + throw new ArgumentNullException (nameof(option)); + try { + return base [option]; + } + catch (KeyNotFoundException) { + return null; } + } + protected override void InsertItem (int index, Option item) + { + base.InsertItem (index, item); + AddImpl (item); + } - protected override string GetKeyForItem (Option item) - { - if (item == null) - throw new ArgumentNullException ("option"); - if (item.Names != null && item.Names.Length > 0) - return item.Names [0]; - // This should never happen, as it's invalid for Option to be - // constructed w/o any names. - throw new InvalidOperationException ("Option has no names!"); + protected override void RemoveItem (int index) + { + Option p = Items [index]; + base.RemoveItem (index); + // KeyedCollection.RemoveItem() handles the 0th item + for (int i = 1; i < p.Names.Length; ++i) { + Dictionary.Remove (p.Names [i]); } + } - [Obsolete ("Use KeyedCollection.this[string]")] - protected Option GetOptionForName (string option) - { - if (option == null) - throw new ArgumentNullException (nameof(option)); - try { - return base [option]; - } - catch (KeyNotFoundException) { - return null; + protected override void SetItem (int index, Option item) + { + base.SetItem (index, item); + AddImpl (item); + } + + private void AddImpl (Option option) + { + if (option == null) + throw new ArgumentNullException (nameof(option)); + List added = new List (option.Names.Length); + try { + // KeyedCollection.InsertItem/SetItem handle the 0th name. + for (int i = 1; i < option.Names.Length; ++i) { + Dictionary.Add (option.Names [i], option); + added.Add (option.Names [i]); } } + catch (Exception) { + foreach (string name in added) + Dictionary.Remove (name); + throw; + } + } + + public OptionSet Add (string header) + { + if (header == null) + throw new ArgumentNullException (nameof(header)); + Add (new Category (header)); + return this; + } - protected override void InsertItem (int index, Option item) + internal sealed class Category : Option { + + // Prototype starts with '=' because this is an invalid prototype + // (see Option.ParsePrototype(), and thus it'll prevent Category + // instances from being accidentally used as normal options. + public Category (string description) + : base ("=:Category:= " + description, description) { - base.InsertItem (index, item); - AddImpl (item); } - protected override void RemoveItem (int index) + protected override void OnParseComplete (OptionContext c) { - Option p = Items [index]; - base.RemoveItem (index); - // KeyedCollection.RemoveItem() handles the 0th item - for (int i = 1; i < p.Names.Length; ++i) { - Dictionary.Remove (p.Names [i]); - } + throw new NotSupportedException ("Category.OnParseComplete should not be invoked."); } + } - protected override void SetItem (int index, Option item) + + public new OptionSet Add (Option option) + { + base.Add (option); + return this; + } + + sealed class ActionOption : Option { + Action action; + + public ActionOption (string prototype, string description, int count, Action action) + : this (prototype, description, count, action, false) { - base.SetItem (index, item); - AddImpl (item); } - private void AddImpl (Option option) + public ActionOption (string prototype, string description, int count, Action action, bool hidden) + : base (prototype, description, count, hidden) { - if (option == null) - throw new ArgumentNullException (nameof(option)); - List added = new List (option.Names.Length); - try { - // KeyedCollection.InsertItem/SetItem handle the 0th name. - for (int i = 1; i < option.Names.Length; ++i) { - Dictionary.Add (option.Names [i], option); - added.Add (option.Names [i]); - } - } - catch (Exception) { - foreach (string name in added) - Dictionary.Remove (name); - throw; - } + this.action = action ?? throw new ArgumentNullException (nameof(action)); } - public OptionSet Add (string header) + protected override void OnParseComplete (OptionContext c) { - if (header == null) - throw new ArgumentNullException (nameof(header)); - Add (new Category (header)); - return this; + action (c.OptionValues); } + } - internal sealed class Category : Option { + public OptionSet Add (string prototype, Action action) + { + return Add (prototype, null, action); + } - // Prototype starts with '=' because this is an invalid prototype - // (see Option.ParsePrototype(), and thus it'll prevent Category - // instances from being accidentally used as normal options. - public Category (string description) - : base ("=:Category:= " + description, description) - { - } + public OptionSet Add (string prototype, string description, Action action) + { + return Add (prototype, description, action, false); + } - protected override void OnParseComplete (OptionContext c) + public OptionSet Add (string prototype, string description, Action action, bool hidden) + { + if (action == null) + throw new ArgumentNullException (nameof(action)); + Option p = new ActionOption (prototype, description, 1, + delegate (OptionValueCollection v) { - throw new NotSupportedException ("Category.OnParseComplete should not be invoked."); - } - } - - - public new OptionSet Add (Option option) - { - base.Add (option); - return this; - } + var v0 = v[0]; + if (!string.IsNullOrWhiteSpace(v0)) + { + action(v0); + } + }, hidden); + base.Add (p); + return this; + } - sealed class ActionOption : Option { - Action action; + public OptionSet Add (string prototype, OptionAction action) + { + return Add (prototype, null, action); + } - public ActionOption (string prototype, string description, int count, Action action) - : this (prototype, description, count, action, false) - { - } + public OptionSet Add (string prototype, string description, OptionAction action) + { + return Add (prototype, description, action, false); + } - public ActionOption (string prototype, string description, int count, Action action, bool hidden) - : base (prototype, description, count, hidden) - { - this.action = action ?? throw new ArgumentNullException (nameof(action)); - } + public OptionSet Add (string prototype, string description, OptionAction action, bool hidden) { + if (action == null) + throw new ArgumentNullException (nameof(action)); + Option p = new ActionOption (prototype, description, 2, + delegate (OptionValueCollection v) {action (v [0], v [1]);}, hidden); + base.Add (p); + return this; + } - protected override void OnParseComplete (OptionContext c) - { - action (c.OptionValues); - } - } + sealed class ActionOption : Option { + Action action; - public OptionSet Add (string prototype, Action action) + public ActionOption (string prototype, string description, Action action) + : base (prototype, description, 1) { - return Add (prototype, null, action); + this.action = action ?? throw new ArgumentNullException (nameof(action)); } - public OptionSet Add (string prototype, string description, Action action) + protected override void OnParseComplete (OptionContext c) { - return Add (prototype, description, action, false); + action (Parse (c.OptionValues [0], c)); } + } - public OptionSet Add (string prototype, string description, Action action, bool hidden) - { - if (action == null) - throw new ArgumentNullException (nameof(action)); - Option p = new ActionOption (prototype, description, 1, - delegate (OptionValueCollection v) - { - var v0 = v[0]; - if (!string.IsNullOrWhiteSpace(v0)) - { - action(v0); - } - }, hidden); - base.Add (p); - return this; - } + sealed class ActionOption : Option { + OptionAction action; - public OptionSet Add (string prototype, OptionAction action) + public ActionOption (string prototype, string description, OptionAction action) + : base (prototype, description, 2) { - return Add (prototype, null, action); + this.action = action ?? throw new ArgumentNullException (nameof(action)); } - public OptionSet Add (string prototype, string description, OptionAction action) + protected override void OnParseComplete (OptionContext c) { - return Add (prototype, description, action, false); + action ( + Parse (c.OptionValues [0], c), + Parse (c.OptionValues [1], c)); } + } - public OptionSet Add (string prototype, string description, OptionAction action, bool hidden) { - if (action == null) - throw new ArgumentNullException (nameof(action)); - Option p = new ActionOption (prototype, description, 2, - delegate (OptionValueCollection v) {action (v [0], v [1]);}, hidden); - base.Add (p); - return this; - } + public OptionSet Add (string prototype, Action action) + { + return Add (prototype, null, action); + } - sealed class ActionOption : Option { - Action action; + public OptionSet Add (string prototype, string description, Action action) + { + return Add (new ActionOption (prototype, description, action)); + } - public ActionOption (string prototype, string description, Action action) - : base (prototype, description, 1) - { - this.action = action ?? throw new ArgumentNullException (nameof(action)); - } + public OptionSet Add (string prototype, OptionAction action) + { + return Add (prototype, null, action); + } - protected override void OnParseComplete (OptionContext c) - { - action (Parse (c.OptionValues [0], c)); - } - } + public OptionSet Add (string prototype, string description, OptionAction action) + { + return Add (new ActionOption (prototype, description, action)); + } - sealed class ActionOption : Option { - OptionAction action; + public OptionSet Add (ArgumentSource source) + { + if (source == null) + throw new ArgumentNullException (nameof(source)); + sources.Add (source); + return this; + } - public ActionOption (string prototype, string description, OptionAction action) - : base (prototype, description, 2) - { - this.action = action ?? throw new ArgumentNullException (nameof(action)); - } + protected virtual OptionContext CreateOptionContext () + { + return new OptionContext (this); + } - protected override void OnParseComplete (OptionContext c) - { - action ( - Parse (c.OptionValues [0], c), - Parse (c.OptionValues [1], c)); + public List Parse (IEnumerable arguments) + { + if (arguments == null) + throw new ArgumentNullException (nameof(arguments)); + OptionContext c = CreateOptionContext (); + c.OptionIndex = -1; + bool process = true; + List unprocessed = []; + Option def = Contains ("<>") ? this ["<>"] : null; + ArgumentEnumerator ae = new ArgumentEnumerator (arguments); + foreach (string argument in ae) { + ++c.OptionIndex; + if (argument == "--") { + process = false; + continue; } + if (!process) { + Unprocessed (unprocessed, def, c, argument); + continue; + } + if (AddSource (ae, argument)) + continue; + if (!Parse (argument, c)) + Unprocessed (unprocessed, def, c, argument); } + if (c.Option != null) + c.Option.Invoke (c); + return unprocessed; + } - public OptionSet Add (string prototype, Action action) - { - return Add (prototype, null, action); - } + class ArgumentEnumerator : IEnumerable { + List> sources = []; - public OptionSet Add (string prototype, string description, Action action) + public ArgumentEnumerator (IEnumerable arguments) { - return Add (new ActionOption (prototype, description, action)); + sources.Add (arguments.GetEnumerator ()); } - public OptionSet Add (string prototype, OptionAction action) + public void Add (IEnumerable arguments) { - return Add (prototype, null, action); + sources.Add (arguments.GetEnumerator ()); } - public OptionSet Add (string prototype, string description, OptionAction action) + public IEnumerator GetEnumerator () { - return Add (new ActionOption (prototype, description, action)); + do { + IEnumerator c = sources [sources.Count-1]; + if (c.MoveNext ()) + yield return c.Current; + else { + c.Dispose (); + sources.RemoveAt (sources.Count-1); + } + } while (sources.Count > 0); } - public OptionSet Add (ArgumentSource source) + IEnumerator IEnumerable.GetEnumerator () { - if (source == null) - throw new ArgumentNullException (nameof(source)); - sources.Add (source); - return this; + return GetEnumerator (); } + } - protected virtual OptionContext CreateOptionContext () - { - return new OptionContext (this); + bool AddSource (ArgumentEnumerator ae, string argument) + { + foreach (ArgumentSource source in sources) { + IEnumerable replacement; + if (!source.GetArguments (argument, out replacement)) + continue; + ae.Add (replacement); + return true; } + return false; + } - public List Parse (IEnumerable arguments) - { - if (arguments == null) - throw new ArgumentNullException (nameof(arguments)); - OptionContext c = CreateOptionContext (); - c.OptionIndex = -1; - bool process = true; - List unprocessed = []; - Option def = Contains ("<>") ? this ["<>"] : null; - ArgumentEnumerator ae = new ArgumentEnumerator (arguments); - foreach (string argument in ae) { - ++c.OptionIndex; - if (argument == "--") { - process = false; - continue; - } - if (!process) { - Unprocessed (unprocessed, def, c, argument); - continue; - } - if (AddSource (ae, argument)) - continue; - if (!Parse (argument, c)) - Unprocessed (unprocessed, def, c, argument); - } - if (c.Option != null) - c.Option.Invoke (c); - return unprocessed; + private static bool Unprocessed (ICollection extra, Option def, OptionContext c, string argument) + { + if (def == null) { + extra.Add (argument); + return false; } + c.OptionValues.Add (argument); + c.Option = def; + c.Option.Invoke (c); + return false; + } - class ArgumentEnumerator : IEnumerable { - List> sources = []; - - public ArgumentEnumerator (IEnumerable arguments) - { - sources.Add (arguments.GetEnumerator ()); - } - - public void Add (IEnumerable arguments) - { - sources.Add (arguments.GetEnumerator ()); - } - - public IEnumerator GetEnumerator () - { - do { - IEnumerator c = sources [sources.Count-1]; - if (c.MoveNext ()) - yield return c.Current; - else { - c.Dispose (); - sources.RemoveAt (sources.Count-1); - } - } while (sources.Count > 0); - } + private readonly Regex ValueOption = new( + @"^(?--|-|/)(?[^:=]+)((?[:=])(?.*))?$"); - IEnumerator IEnumerable.GetEnumerator () - { - return GetEnumerator (); - } - } + protected bool GetOptionParts (string argument, out string flag, out string name, out string sep, out string value) + { + if (argument == null) + throw new ArgumentNullException (nameof(argument)); - bool AddSource (ArgumentEnumerator ae, string argument) - { - foreach (ArgumentSource source in sources) { - IEnumerable replacement; - if (!source.GetArguments (argument, out replacement)) - continue; - ae.Add (replacement); - return true; - } + flag = name = sep = value = null; + Match m = ValueOption.Match (argument); + if (!m.Success) { return false; } - - private static bool Unprocessed (ICollection extra, Option def, OptionContext c, string argument) - { - if (def == null) { - extra.Add (argument); - return false; - } - c.OptionValues.Add (argument); - c.Option = def; - c.Option.Invoke (c); - return false; + flag = m.Groups ["flag"].Value; + name = m.Groups ["name"].Value; + if (m.Groups ["sep"].Success && m.Groups ["value"].Success) { + sep = m.Groups ["sep"].Value; + value = m.Groups ["value"].Value; } + return true; + } - private readonly Regex ValueOption = new Regex ( - @"^(?--|-|/)(?[^:=]+)((?[:=])(?.*))?$"); + protected virtual bool Parse (string argument, OptionContext c) + { + if (c.Option != null) { + ParseValue (argument, c); + return true; + } - protected bool GetOptionParts (string argument, out string flag, out string name, out string sep, out string value) - { - if (argument == null) - throw new ArgumentNullException (nameof(argument)); + string f, n, s, v; + if (!GetOptionParts (argument, out f, out n, out s, out v)) + return false; - flag = name = sep = value = null; - Match m = ValueOption.Match (argument); - if (!m.Success) { - return false; - } - flag = m.Groups ["flag"].Value; - name = m.Groups ["name"].Value; - if (m.Groups ["sep"].Success && m.Groups ["value"].Success) { - sep = m.Groups ["sep"].Value; - value = m.Groups ["value"].Value; + Option p; + if (Contains (n)) { + p = this [n]; + c.OptionName = f + n; + c.Option = p; + switch (p.OptionValueType) { + case OptionValueType.None: + c.OptionValues.Add (n); + c.Option.Invoke (c); + break; + case OptionValueType.Optional: + case OptionValueType.Required: + ParseValue (v, c); + break; } return true; } + // no match; is it a bool option? + if (ParseBool (argument, n, c)) + return true; + // is it a bundled option? + if (ParseBundledValue (f, string.Concat (n + s + v), c)) + return true; - protected virtual bool Parse (string argument, OptionContext c) - { - if (c.Option != null) { - ParseValue (argument, c); - return true; - } - - string f, n, s, v; - if (!GetOptionParts (argument, out f, out n, out s, out v)) - return false; + return false; + } - Option p; - if (Contains (n)) { - p = this [n]; - c.OptionName = f + n; - c.Option = p; - switch (p.OptionValueType) { - case OptionValueType.None: - c.OptionValues.Add (n); - c.Option.Invoke (c); - break; - case OptionValueType.Optional: - case OptionValueType.Required: - ParseValue (v, c); - break; - } - return true; + private void ParseValue (string option, OptionContext c) + { + if (option != null) + foreach (string o in c.Option.ValueSeparators != null + ? option.Split (c.Option.ValueSeparators, c.Option.MaxValueCount - c.OptionValues.Count, StringSplitOptions.None) + : [option]) { + c.OptionValues.Add (o); } - // no match; is it a bool option? - if (ParseBool (argument, n, c)) - return true; - // is it a bundled option? - if (ParseBundledValue (f, string.Concat (n + s + v), c)) - return true; - - return false; + if (c.OptionValues.Count == c.Option.MaxValueCount || + c.Option.OptionValueType == OptionValueType.Optional) + c.Option.Invoke (c); + else if (c.OptionValues.Count > c.Option.MaxValueCount) { + throw new OptionException (localizer (string.Format ( + "Error: Found {0} option values when expecting {1}.", + c.OptionValues.Count, c.Option.MaxValueCount)), + c.OptionName); } + } - private void ParseValue (string option, OptionContext c) - { - if (option != null) - foreach (string o in c.Option.ValueSeparators != null - ? option.Split (c.Option.ValueSeparators, c.Option.MaxValueCount - c.OptionValues.Count, StringSplitOptions.None) - : [option]) { - c.OptionValues.Add (o); - } - if (c.OptionValues.Count == c.Option.MaxValueCount || - c.Option.OptionValueType == OptionValueType.Optional) - c.Option.Invoke (c); - else if (c.OptionValues.Count > c.Option.MaxValueCount) { - throw new OptionException (localizer (string.Format ( - "Error: Found {0} option values when expecting {1}.", - c.OptionValues.Count, c.Option.MaxValueCount)), - c.OptionName); - } + private bool ParseBool (string option, string n, OptionContext c) + { + Option p; + string rn; + if (n.Length >= 1 && (n [n.Length-1] == '+' || n [n.Length-1] == '-') && + Contains ((rn = n.Substring (0, n.Length-1)))) { + p = this [rn]; + string v = n [n.Length-1] == '+' ? option : null; + c.OptionName = option; + c.Option = p; + c.OptionValues.Add (v); + p.Invoke (c); + return true; } + return false; + } - private bool ParseBool (string option, string n, OptionContext c) - { + private bool ParseBundledValue (string f, string n, OptionContext c) + { + if (f != "-") + return false; + for (int i = 0; i < n.Length; ++i) { Option p; - string rn; - if (n.Length >= 1 && (n [n.Length-1] == '+' || n [n.Length-1] == '-') && - Contains ((rn = n.Substring (0, n.Length-1)))) { - p = this [rn]; - string v = n [n.Length-1] == '+' ? option : null; - c.OptionName = option; - c.Option = p; - c.OptionValues.Add (v); - p.Invoke (c); - return true; + string opt = f + n [i].ToString (); + string rn = n [i].ToString (); + if (!Contains (rn)) { + if (i == 0) + return false; + throw new OptionException (string.Format (localizer ( + "Cannot bundle unregistered option '{0}'."), opt), opt); } - return false; - } - - private bool ParseBundledValue (string f, string n, OptionContext c) - { - if (f != "-") - return false; - for (int i = 0; i < n.Length; ++i) { - Option p; - string opt = f + n [i].ToString (); - string rn = n [i].ToString (); - if (!Contains (rn)) { - if (i == 0) - return false; - throw new OptionException (string.Format (localizer ( - "Cannot bundle unregistered option '{0}'."), opt), opt); - } - p = this [rn]; - switch (p.OptionValueType) { - case OptionValueType.None: - Invoke (c, opt, n, p); - break; - case OptionValueType.Optional: - case OptionValueType.Required: { - string v = n.Substring (i+1); - c.Option = p; - c.OptionName = opt; - ParseValue (v.Length != 0 ? v : null, c); - return true; - } - default: - throw new InvalidOperationException ("Unknown OptionValueType: " + p.OptionValueType); + p = this [rn]; + switch (p.OptionValueType) { + case OptionValueType.None: + Invoke (c, opt, n, p); + break; + case OptionValueType.Optional: + case OptionValueType.Required: { + string v = n.Substring (i+1); + c.Option = p; + c.OptionName = opt; + ParseValue (v.Length != 0 ? v : null, c); + return true; } + default: + throw new InvalidOperationException ("Unknown OptionValueType: " + p.OptionValueType); } - return true; - } - - private static void Invoke (OptionContext c, string name, string value, Option option) - { - c.OptionName = name; - c.Option = option; - c.OptionValues.Add (value); - option.Invoke (c); } + return true; + } - private const int OptionWidth = 29; - private const int Description_FirstWidth = 80 - OptionWidth; - private const int Description_RemWidth = 80 - OptionWidth - 2; + private static void Invoke (OptionContext c, string name, string value, Option option) + { + c.OptionName = name; + c.Option = option; + c.OptionValues.Add (value); + option.Invoke (c); + } - public void WriteOptionDescriptions (TextWriter o) - { - foreach (Option p in this) { - int written = 0; + private const int OptionWidth = 29; + private const int Description_FirstWidth = 80 - OptionWidth; + private const int Description_RemWidth = 80 - OptionWidth - 2; - if (p.Hidden) - continue; + public void WriteOptionDescriptions (TextWriter o) + { + foreach (Option p in this) { + int written = 0; - Category c = p as Category; - if (c != null) { - WriteDescription (o, p.Description, "", 80, 80); - continue; - } + if (p.Hidden) + continue; - if (!WriteOptionPrototype (o, p, ref written)) - continue; + Category c = p as Category; + if (c != null) { + WriteDescription (o, p.Description, "", 80, 80); + continue; + } - if (written < OptionWidth) - o.Write (new string (' ', OptionWidth - written)); - else { - o.WriteLine (); - o.Write (new string (' ', OptionWidth)); - } + if (!WriteOptionPrototype (o, p, ref written)) + continue; - WriteDescription (o, p.Description, new string (' ', OptionWidth+2), - Description_FirstWidth -1, Description_RemWidth - 2); + if (written < OptionWidth) + o.Write (new string (' ', OptionWidth - written)); + else { + o.WriteLine (); + o.Write (new string (' ', OptionWidth)); } - foreach (ArgumentSource s in sources) { - string[] names = s.GetNames (); - if (names == null || names.Length == 0) - continue; + WriteDescription (o, p.Description, new string (' ', OptionWidth+2), + Description_FirstWidth -1, Description_RemWidth - 2); + } - int written = 0; + foreach (ArgumentSource s in sources) { + string[] names = s.GetNames (); + if (names == null || names.Length == 0) + continue; - Write (o, ref written, " "); - Write (o, ref written, names [0]); - for (int i = 1; i < names.Length; ++i) { - Write (o, ref written, ", "); - Write (o, ref written, names [i]); - } + int written = 0; - if (written < OptionWidth) - o.Write (new string (' ', OptionWidth - written)); - else { - o.WriteLine (); - o.Write (new string (' ', OptionWidth)); - } - - WriteDescription (o, s.Description, new string (' ', OptionWidth+2), - Description_FirstWidth, Description_RemWidth); + Write (o, ref written, " "); + Write (o, ref written, names [0]); + for (int i = 1; i < names.Length; ++i) { + Write (o, ref written, ", "); + Write (o, ref written, names [i]); } - } - void WriteDescription (TextWriter o, string value, string prefix, int firstWidth, int remWidth) - { - bool indent = false; - foreach (string line in GetLines (localizer (GetDescription (value)), firstWidth, remWidth)) { - if (indent) - o.Write (prefix); - o.WriteLine (line); - indent = true; + if (written < OptionWidth) + o.Write (new string (' ', OptionWidth - written)); + else { + o.WriteLine (); + o.Write (new string (' ', OptionWidth)); } + + WriteDescription (o, s.Description, new string (' ', OptionWidth+2), + Description_FirstWidth, Description_RemWidth); } + } - public bool WriteOptionPrototype (TextWriter o, Option p, ref int written, bool markdown = false) - { - string[] names = p.Names; - - int i = GetNextOptionIndex (names, 0); - if (i == names.Length) - return false; - - if (names [i].Length == 1) { - if (markdown) - Write(o, ref written, "`-"); - else - Write(o, ref written, " -"); - Write(o, ref written, names [0]); - } - else { - if (markdown) - Write(o, ref written, " `--"); - else - Write (o, ref written, " --"); - Write (o, ref written, names[0]); - } - - for ( i = GetNextOptionIndex (names, i+1); - i < names.Length; i = GetNextOptionIndex (names, i+1)) { - if (markdown) - Write(o, ref written, "`, `"); - else - Write(o, ref written, ", "); - Write (o, ref written, names [i].Length == 1 ? "-" : "--"); - Write (o, ref written, names [i]); - } + void WriteDescription (TextWriter o, string value, string prefix, int firstWidth, int remWidth) + { + bool indent = false; + foreach (string line in GetLines (localizer (GetDescription (value)), firstWidth, remWidth)) { + if (indent) + o.Write (prefix); + o.WriteLine (line); + indent = true; + } + } - if (p.OptionValueType == OptionValueType.Optional || - p.OptionValueType == OptionValueType.Required) { - if (p.OptionValueType == OptionValueType.Optional) { - Write (o, ref written, localizer ("[")); - } - Write (o, ref written, localizer ("=" + GetArgumentName (0, p.MaxValueCount, p.Description))); - string sep = p.ValueSeparators != null && p.ValueSeparators.Length > 0 - ? p.ValueSeparators [0] - : " "; - for (int c = 1; c < p.MaxValueCount; ++c) { - Write (o, ref written, localizer (sep + GetArgumentName (c, p.MaxValueCount, p.Description))); - } - if (p.OptionValueType == OptionValueType.Optional) { - Write (o, ref written, localizer ("]")); - } - } + public bool WriteOptionPrototype (TextWriter o, Option p, ref int written, bool markdown = false) + { + string[] names = p.Names; - if (markdown) - Write(o, ref written, "`"); + int i = GetNextOptionIndex (names, 0); + if (i == names.Length) + return false; - return true; + if (names [i].Length == 1) { + if (markdown) + Write(o, ref written, "`-"); + else + Write(o, ref written, " -"); + Write(o, ref written, names [0]); + } + else { + if (markdown) + Write(o, ref written, " `--"); + else + Write (o, ref written, " --"); + Write (o, ref written, names[0]); + } + + for ( i = GetNextOptionIndex (names, i+1); + i < names.Length; i = GetNextOptionIndex (names, i+1)) { + if (markdown) + Write(o, ref written, "`, `"); + else + Write(o, ref written, ", "); + Write (o, ref written, names [i].Length == 1 ? "-" : "--"); + Write (o, ref written, names [i]); } - static int GetNextOptionIndex (string[] names, int i) - { - while (i < names.Length && names [i] == "<>") { - ++i; + if (p.OptionValueType == OptionValueType.Optional || + p.OptionValueType == OptionValueType.Required) { + if (p.OptionValueType == OptionValueType.Optional) { + Write (o, ref written, localizer ("[")); + } + Write (o, ref written, localizer ("=" + GetArgumentName (0, p.MaxValueCount, p.Description))); + string sep = p.ValueSeparators != null && p.ValueSeparators.Length > 0 + ? p.ValueSeparators [0] + : " "; + for (int c = 1; c < p.MaxValueCount; ++c) { + Write (o, ref written, localizer (sep + GetArgumentName (c, p.MaxValueCount, p.Description))); + } + if (p.OptionValueType == OptionValueType.Optional) { + Write (o, ref written, localizer ("]")); } - return i; } - static void Write (TextWriter o, ref int n, string s) - { - n += s.Length; - o.Write (s); + if (markdown) + Write(o, ref written, "`"); + + return true; + } + + static int GetNextOptionIndex (string[] names, int i) + { + while (i < names.Length && names [i] == "<>") { + ++i; } + return i; + } - private static string GetArgumentName (int index, int maxIndex, string description) - { - if (description == null) - return maxIndex == 1 ? "VALUE" : index == 0 ? "NAME" : "VALUE"; - string[] nameStart; - if (maxIndex == 1) - nameStart = ["{0:", "{"]; - else - nameStart = ["{" + index + ":"]; - for (int i = 0; i < nameStart.Length; ++i) { - int start, j = 0; - do { - start = description.IndexOf (nameStart [i], j); - } while (start >= 0 && j != 0 ? description [j++ - 1] == '{' : false); - if (start == -1) - continue; - int end = description.IndexOf ("}", start); - if (end == -1) - continue; - return description.Substring (start + nameStart [i].Length, end - start - nameStart [i].Length); - } + static void Write (TextWriter o, ref int n, string s) + { + n += s.Length; + o.Write (s); + } + + private static string GetArgumentName (int index, int maxIndex, string description) + { + if (description == null) return maxIndex == 1 ? "VALUE" : index == 0 ? "NAME" : "VALUE"; - } + string[] nameStart; + if (maxIndex == 1) + nameStart = ["{0:", "{"]; + else + nameStart = ["{" + index + ":"]; + for (int i = 0; i < nameStart.Length; ++i) { + int start, j = 0; + do { + start = description.IndexOf (nameStart [i], j); + } while (start >= 0 && j != 0 ? description [j++ - 1] == '{' : false); + if (start == -1) + continue; + int end = description.IndexOf ("}", start); + if (end == -1) + continue; + return description.Substring (start + nameStart [i].Length, end - start - nameStart [i].Length); + } + return maxIndex == 1 ? "VALUE" : index == 0 ? "NAME" : "VALUE"; + } - private static string GetDescription (string description) - { - if (description == null) - return string.Empty; - StringBuilder sb = new StringBuilder (description.Length); - int start = -1; - for (int i = 0; i < description.Length; ++i) { - switch (description [i]) { - case '{': - if (i == start) { - sb.Append ('{'); - start = -1; - } - else if (start < 0) - start = i + 1; - break; - case '}': - if (start < 0) { - if ((i+1) == description.Length || description [i+1] != '}') - throw new InvalidOperationException ("Invalid option description: " + description); - ++i; - sb.Append ("}"); - } - else { - sb.Append (description.Substring (start, i - start)); - start = -1; - } - break; - case ':': - if (start < 0) - goto default; + private static string GetDescription (string description) + { + if (description == null) + return string.Empty; + StringBuilder sb = new StringBuilder (description.Length); + int start = -1; + for (int i = 0; i < description.Length; ++i) { + switch (description [i]) { + case '{': + if (i == start) { + sb.Append ('{'); + start = -1; + } + else if (start < 0) start = i + 1; - break; - default: - if (start < 0) - sb.Append (description [i]); - break; - } + break; + case '}': + if (start < 0) { + if ((i+1) == description.Length || description [i+1] != '}') + throw new InvalidOperationException ("Invalid option description: " + description); + ++i; + sb.Append ("}"); + } + else { + sb.Append (description.Substring (start, i - start)); + start = -1; + } + break; + case ':': + if (start < 0) + goto default; + start = i + 1; + break; + default: + if (start < 0) + sb.Append (description [i]); + break; } - return sb.ToString (); - } - - private static IEnumerable GetLines (string description, int firstWidth, int remWidth) - { - return StringCoda.WrappedLines (description, firstWidth, remWidth); } + return sb.ToString (); } -} + private static IEnumerable GetLines (string description, int firstWidth, int remWidth) + { + return StringCoda.WrappedLines (description, firstWidth, remWidth); + } +} \ No newline at end of file diff --git a/src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs index 3c0291a6..c0c27c19 100644 --- a/src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs +++ b/src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs @@ -1,5 +1,7 @@ namespace SeqCli.Config.Forwarder; +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global + class ForwarderApiConfig { public string ListenUri { get; set; } = "http://127.0.0.1:15341"; diff --git a/src/SeqCli/Config/Forwarder/ForwarderConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderConfig.cs index fac69ad4..cbb2e16f 100644 --- a/src/SeqCli/Config/Forwarder/ForwarderConfig.cs +++ b/src/SeqCli/Config/Forwarder/ForwarderConfig.cs @@ -1,5 +1,7 @@ namespace SeqCli.Config.Forwarder; +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global + class ForwarderConfig { public ForwarderStorageConfig Storage { get; set; } = new(); diff --git a/src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs index 1edc44fe..3a63d685 100644 --- a/src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs +++ b/src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs @@ -1,6 +1,8 @@ using System; using System.IO; using Serilog.Events; +// ReSharper disable UnusedAutoPropertyAccessor.Global +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global namespace SeqCli.Config.Forwarder; diff --git a/src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs index 455daacb..5bc58044 100644 --- a/src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs +++ b/src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs @@ -1,5 +1,7 @@ namespace SeqCli.Config.Forwarder; +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global + public class ForwarderStorageConfig { public ulong BufferSizeBytes { get; set; } = 67_108_864; diff --git a/src/SeqCli/Config/OutputConfig.cs b/src/SeqCli/Config/OutputConfig.cs index 8727c2b3..b52a73b6 100644 --- a/src/SeqCli/Config/OutputConfig.cs +++ b/src/SeqCli/Config/OutputConfig.cs @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +// ReSharper disable UnusedAutoPropertyAccessor.Global +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global + namespace SeqCli.Config; public class OutputConfig diff --git a/src/SeqCli/Config/SeqCliConfig.cs b/src/SeqCli/Config/SeqCliConfig.cs index 93160e69..88ac1b08 100644 --- a/src/SeqCli/Config/SeqCliConfig.cs +++ b/src/SeqCli/Config/SeqCliConfig.cs @@ -19,6 +19,8 @@ using Newtonsoft.Json.Converters; using Newtonsoft.Json.Serialization; using SeqCli.Config.Forwarder; +// ReSharper disable UnusedAutoPropertyAccessor.Global +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global namespace SeqCli.Config; @@ -27,7 +29,7 @@ class SeqCliConfig static readonly string DefaultConfigFilename = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "SeqCli.json"); - static JsonSerializerSettings SerializerSettings { get; } = new JsonSerializerSettings + static JsonSerializerSettings SerializerSettings { get; } = new() { ContractResolver = new CamelCasePropertyNamesContractResolver(), Converters = @@ -52,11 +54,10 @@ public static void Write(SeqCliConfig data) File.WriteAllText(DefaultConfigFilename, content); } - public ConnectionConfig Connection { get; set; } = new ConnectionConfig(); + public ConnectionConfig Connection { get; set; } = new(); public OutputConfig Output { get; set; } = new(); public ForwarderConfig Forwarder { get; set; } = new(); public SeqCliEncryptionProviderConfig EncryptionProviderProvider { get; set; } = new SeqCliEncryptionProviderConfig(); - public Dictionary Profiles { get; } = - new Dictionary(StringComparer.OrdinalIgnoreCase); + public Dictionary Profiles { get; } = new(StringComparer.OrdinalIgnoreCase); } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs b/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs index 0d49b92e..cdc930c1 100644 --- a/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs +++ b/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs @@ -1,8 +1,7 @@ -namespace SeqCli.Forwarder.Cryptography +namespace SeqCli.Forwarder.Cryptography; + +public interface IStringDataProtector { - public interface IStringDataProtector - { - string Protect(string value); - string Unprotect(string @protected); - } -} + string Protect(string value); + string Unprotect(string @protected); +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs b/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs index 46bd1ae2..e35ef0b7 100644 --- a/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs +++ b/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs @@ -1,14 +1,13 @@ -namespace SeqCli.Forwarder.Cryptography +namespace SeqCli.Forwarder.Cryptography; + +static class StringDataProtector { - static class StringDataProtector + public static IStringDataProtector CreatePlatformDefault() { - public static IStringDataProtector CreatePlatformDefault() - { #if WINDOWS return new DpapiMachineScopeDataProtect(); #else - return new UnprotectedStringData(); + return new UnprotectedStringData(); #endif - } } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs b/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs index ef556083..6148081e 100644 --- a/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs +++ b/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs @@ -2,20 +2,19 @@ using Serilog; -namespace SeqCli.Forwarder.Cryptography +namespace SeqCli.Forwarder.Cryptography; + +public class UnprotectedStringData : IStringDataProtector { - public class UnprotectedStringData : IStringDataProtector + public string Protect(string value) { - public string Protect(string value) - { - Log.Warning("Data protection is not available on this platform; sensitive values will be stored in plain text"); - return value; - } + Log.Warning("Data protection is not available on this platform; sensitive values will be stored in plain text"); + return value; + } - public string Unprotect(string @protected) - { - return @protected; - } + public string Unprotect(string @protected) + { + return @protected; } } diff --git a/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs b/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs index 38d0c6db..b5eea21f 100644 --- a/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs +++ b/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs @@ -18,32 +18,31 @@ using Serilog.Core; using Serilog.Events; -namespace SeqCli.Forwarder.Diagnostics +namespace SeqCli.Forwarder.Diagnostics; + +public class InMemorySink : ILogEventSink { - public class InMemorySink : ILogEventSink - { - readonly int _queueLength; - readonly ConcurrentQueue _queue = new ConcurrentQueue(); + readonly int _queueLength; + readonly ConcurrentQueue _queue = new(); - public InMemorySink(int queueLength) - { - _queueLength = queueLength; - } + public InMemorySink(int queueLength) + { + _queueLength = queueLength; + } - public IEnumerable Read() - { - return _queue.ToArray(); - } + public IEnumerable Read() + { + return _queue.ToArray(); + } - public void Emit(LogEvent logEvent) - { - if (logEvent == null) throw new ArgumentNullException(nameof(logEvent)); - _queue.Enqueue(logEvent); + public void Emit(LogEvent logEvent) + { + if (logEvent == null) throw new ArgumentNullException(nameof(logEvent)); + _queue.Enqueue(logEvent); - while (_queue.Count > _queueLength) - { - _queue.TryDequeue(out _); - } + while (_queue.Count > _queueLength) + { + _queue.TryDequeue(out _); } } -} +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs b/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs index e2fa7b5c..5e54a5bf 100644 --- a/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs +++ b/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs @@ -18,48 +18,47 @@ using Serilog; using Serilog.Events; -namespace SeqCli.Forwarder.Diagnostics +namespace SeqCli.Forwarder.Diagnostics; + +static class IngestionLog { - static class IngestionLog - { - const int Capacity = 100; + const int Capacity = 100; - static readonly InMemorySink Sink = new InMemorySink(Capacity); + static readonly InMemorySink Sink = new(Capacity); - public static ILogger Log { get; } + public static ILogger Log { get; } - static IngestionLog() - { - Log = new LoggerConfiguration() - .MinimumLevel.Verbose() - .WriteTo.Sink(Sink) - .WriteTo.Logger(Serilog.Log.Logger) - .CreateLogger(); - } + static IngestionLog() + { + Log = new LoggerConfiguration() + .MinimumLevel.Verbose() + .WriteTo.Sink(Sink) + .WriteTo.Logger(Serilog.Log.Logger) + .CreateLogger(); + } - public static IEnumerable Read() - { - return Sink.Read(); - } + public static IEnumerable Read() + { + return Sink.Read(); + } - public static ILogger ForClient(IPAddress clientHostIP) - { - return Log.ForContext("ClientHostIP", clientHostIP); - } + public static ILogger ForClient(IPAddress clientHostIP) + { + return Log.ForContext("ClientHostIP", clientHostIP); + } - public static ILogger ForPayload(IPAddress clientHostIP, string payload) - { - var prefix = CapturePrefix(payload); - return ForClient(clientHostIP) - .ForContext("StartToLog", prefix.Length) - .ForContext("DocumentStart", prefix); - } + public static ILogger ForPayload(IPAddress clientHostIP, string payload) + { + var prefix = CapturePrefix(payload); + return ForClient(clientHostIP) + .ForContext("StartToLog", prefix.Length) + .ForContext("DocumentStart", prefix); + } - static string CapturePrefix(string line) - { - if (line == null) throw new ArgumentNullException(nameof(line)); - var startToLog = Math.Min(line.Length, 1024); - return line.Substring(0, startToLog); - } + static string CapturePrefix(string line) + { + if (line == null) throw new ArgumentNullException(nameof(line)); + var startToLog = Math.Min(line.Length, 1024); + return line.Substring(0, startToLog); } -} +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs new file mode 100644 index 00000000..2a5005ab --- /dev/null +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -0,0 +1,84 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Net.Http; +using System.Threading; +using Autofac; +using SeqCli.Config; +using SeqCli.Forwarder.Cryptography; +using SeqCli.Forwarder.Multiplexing; +using SeqCli.Forwarder.Web.Host; + +namespace SeqCli.Forwarder; + +class ForwarderModule : Module +{ + readonly string _bufferPath; + readonly SeqCliConfig _config; + + public ForwarderModule(string bufferPath, SeqCliConfig config) + { + _bufferPath = bufferPath ?? throw new ArgumentNullException(nameof(bufferPath)); + _config = config ?? throw new ArgumentNullException(nameof(config)); + } + + protected override void Load(ContainerBuilder builder) + { + builder.RegisterType().SingleInstance(); + builder.RegisterType() + .WithParameter("bufferPath", _bufferPath) + .SingleInstance(); + + builder.RegisterType().As(); + builder.RegisterType().SingleInstance(); + + builder.Register(c => + { + var outputConfig = c.Resolve(); + var baseUri = outputConfig.ServerUrl; + if (string.IsNullOrWhiteSpace(baseUri)) + throw new ArgumentException("The destination Seq server URL must be configured in SeqForwarder.json."); + + if (!baseUri.EndsWith("/")) + baseUri += "/"; + + // additional configuration options that require the use of SocketsHttpHandler should be added to + // this expression, using an "or" operator. + + var hasSocketHandlerOption = + outputConfig.PooledConnectionLifetimeMilliseconds.HasValue; + + if (hasSocketHandlerOption) + { + var httpMessageHandler = new SocketsHttpHandler + { + PooledConnectionLifetime = outputConfig.PooledConnectionLifetimeMilliseconds.HasValue ? TimeSpan.FromMilliseconds(outputConfig.PooledConnectionLifetimeMilliseconds.Value) : Timeout.InfiniteTimeSpan, + }; + + return new HttpClient(httpMessageHandler) { BaseAddress = new Uri(baseUri) }; + } + + return new HttpClient { BaseAddress = new Uri(baseUri) }; + + }).SingleInstance(); + + builder.RegisterInstance(StringDataProtector.CreatePlatformDefault()); + + builder.RegisterInstance(_config); + builder.RegisterInstance(_config.Forwarder.Api); + builder.RegisterInstance(_config.Forwarder.Diagnostics); + builder.RegisterInstance(_config.Forwarder.Storage); + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs index ce0f7318..52fd743b 100644 --- a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs +++ b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs @@ -16,23 +16,22 @@ using SeqCli.Forwarder.Shipper; using SeqCli.Forwarder.Storage; -namespace SeqCli.Forwarder.Multiplexing +namespace SeqCli.Forwarder.Multiplexing; + +sealed class ActiveLogBuffer : IDisposable { - sealed class ActiveLogBuffer : IDisposable - { - public LogShipper Shipper { get; } - public LogBuffer Buffer { get; } + public LogShipper Shipper { get; } + public LogBuffer Buffer { get; } - public ActiveLogBuffer(LogBuffer logBuffer, LogShipper logShipper) - { - Buffer = logBuffer ?? throw new ArgumentNullException(nameof(logBuffer)); - Shipper = logShipper ?? throw new ArgumentNullException(nameof(logShipper)); - } + public ActiveLogBuffer(LogBuffer logBuffer, LogShipper logShipper) + { + Buffer = logBuffer ?? throw new ArgumentNullException(nameof(logBuffer)); + Shipper = logShipper ?? throw new ArgumentNullException(nameof(logShipper)); + } - public void Dispose() - { - Shipper.Dispose(); - Buffer.Dispose(); - } + public void Dispose() + { + Shipper.Dispose(); + Buffer.Dispose(); } -} +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs index 47d19db1..bcd09c65 100644 --- a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs +++ b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs @@ -23,201 +23,200 @@ using SeqCli.Forwarder.Web; using Serilog; -namespace SeqCli.Forwarder.Multiplexing +namespace SeqCli.Forwarder.Multiplexing; + +public class ActiveLogBufferMap : IDisposable { - public class ActiveLogBufferMap : IDisposable + const string DataFileName = "data.mdb", LockFileName = "lock.mdb", ApiKeyFileName = ".apikey"; + + readonly ulong _bufferSizeBytes; + readonly ConnectionConfig _connectionConfig; + readonly ILogShipperFactory _shipperFactory; + readonly IStringDataProtector _dataProtector; + readonly string _bufferPath; + readonly ILogger _log = Log.ForContext(); + + readonly object _sync = new(); + bool _loaded; + ActiveLogBuffer? _noApiKeyLogBuffer; + readonly Dictionary _buffersByApiKey = new(); + + public ActiveLogBufferMap( + string bufferPath, + ForwarderStorageConfig storageConfig, + ConnectionConfig outputConfig, + ILogShipperFactory logShipperFactory, + IStringDataProtector dataProtector) { - const string DataFileName = "data.mdb", LockFileName = "lock.mdb", ApiKeyFileName = ".apikey"; - - readonly ulong _bufferSizeBytes; - readonly ConnectionConfig _connectionConfig; - readonly ILogShipperFactory _shipperFactory; - readonly IStringDataProtector _dataProtector; - readonly string _bufferPath; - readonly ILogger _log = Log.ForContext(); - - readonly object _sync = new(); - bool _loaded; - ActiveLogBuffer? _noApiKeyLogBuffer; - readonly Dictionary _buffersByApiKey = new Dictionary(); - - public ActiveLogBufferMap( - string bufferPath, - ForwarderStorageConfig storageConfig, - ConnectionConfig outputConfig, - ILogShipperFactory logShipperFactory, - IStringDataProtector dataProtector) - { - _bufferSizeBytes = storageConfig.BufferSizeBytes; - _connectionConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); - _shipperFactory = logShipperFactory ?? throw new ArgumentNullException(nameof(logShipperFactory)); - _dataProtector = dataProtector ?? throw new ArgumentNullException(nameof(dataProtector)); - _bufferPath = bufferPath ?? throw new ArgumentNullException(nameof(bufferPath)); - } + _bufferSizeBytes = storageConfig.BufferSizeBytes; + _connectionConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); + _shipperFactory = logShipperFactory ?? throw new ArgumentNullException(nameof(logShipperFactory)); + _dataProtector = dataProtector ?? throw new ArgumentNullException(nameof(dataProtector)); + _bufferPath = bufferPath ?? throw new ArgumentNullException(nameof(bufferPath)); + } - // The odd three-stage initialization improves our chances of correctly tearing down the `LightningEnvironment`s within - // `LogBuffer`s in the event of a failure during start-up. See: https://github.com/CoreyKaylor/Lightning.NET/blob/master/src/LightningDB/LightningEnvironment.cs#L252 - public void Load() - { - // At startup, we look for buffers and either delete them if they're empty, or load them - // up if they're not. This garbage collection at start-up is a simplification, - // we might try cleaning up in the background if the gains are worthwhile, although more synchronization - // would be required. + // The odd three-stage initialization improves our chances of correctly tearing down the `LightningEnvironment`s within + // `LogBuffer`s in the event of a failure during start-up. See: https://github.com/CoreyKaylor/Lightning.NET/blob/master/src/LightningDB/LightningEnvironment.cs#L252 + public void Load() + { + // At startup, we look for buffers and either delete them if they're empty, or load them + // up if they're not. This garbage collection at start-up is a simplification, + // we might try cleaning up in the background if the gains are worthwhile, although more synchronization + // would be required. - lock (_sync) - { - if (_loaded) throw new InvalidOperationException("The log buffer map is already loaded."); + lock (_sync) + { + if (_loaded) throw new InvalidOperationException("The log buffer map is already loaded."); - Directory.CreateDirectory(_bufferPath); + Directory.CreateDirectory(_bufferPath); - var defaultDataFilePath = Path.Combine(_bufferPath, DataFileName); - if (File.Exists(defaultDataFilePath)) + var defaultDataFilePath = Path.Combine(_bufferPath, DataFileName); + if (File.Exists(defaultDataFilePath)) + { + _log.Information("Loading the default log buffer in {Path}", _bufferPath); + var buffer = new LogBuffer(_bufferPath, _bufferSizeBytes); + if (buffer.Peek(0).Length == 0) { - _log.Information("Loading the default log buffer in {Path}", _bufferPath); - var buffer = new LogBuffer(_bufferPath, _bufferSizeBytes); - if (buffer.Peek(0).Length == 0) - { - _log.Information("The default buffer is empty and will be removed until more data is received"); - buffer.Dispose(); - File.Delete(defaultDataFilePath); - var lockFilePath = Path.Combine(_bufferPath, LockFileName); - if (File.Exists(lockFilePath)) - File.Delete(lockFilePath); - } - else - { - _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _connectionConfig.GetApiKey(_dataProtector))); - } + _log.Information("The default buffer is empty and will be removed until more data is received"); + buffer.Dispose(); + File.Delete(defaultDataFilePath); + var lockFilePath = Path.Combine(_bufferPath, LockFileName); + if (File.Exists(lockFilePath)) + File.Delete(lockFilePath); } - - foreach (var subfolder in Directory.GetDirectories(_bufferPath)) + else { - var encodedApiKeyFilePath = Path.Combine(subfolder, ApiKeyFileName); - if (!File.Exists(encodedApiKeyFilePath)) - { - _log.Information("Folder {Path} does not appear to be a log buffer; skipping", subfolder); - continue; - } - - _log.Information("Loading an API-key specific buffer in {Path}", subfolder); - var apiKey = _dataProtector.Unprotect(File.ReadAllText(encodedApiKeyFilePath)); - - var buffer = new LogBuffer(subfolder, _bufferSizeBytes); - if (buffer.Peek(0).Length == 0) - { - _log.Information("API key-specific buffer in {Path} is empty and will be removed until more data is received", subfolder); - buffer.Dispose(); - Directory.Delete(subfolder, true); - } - else - { - var activeBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, apiKey)); - _buffersByApiKey.Add(apiKey, activeBuffer); - } + _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _connectionConfig.GetApiKey(_dataProtector))); } - - _loaded = true; } - } - public void Start() - { - lock (_sync) + foreach (var subfolder in Directory.GetDirectories(_bufferPath)) { - if (!_loaded) throw new InvalidOperationException("The log buffer map is not loaded."); - - foreach (var buffer in OpenBuffers) + var encodedApiKeyFilePath = Path.Combine(subfolder, ApiKeyFileName); + if (!File.Exists(encodedApiKeyFilePath)) { - buffer.Shipper.Start(); + _log.Information("Folder {Path} does not appear to be a log buffer; skipping", subfolder); + continue; } - } - } - public void Stop() - { - lock (_sync) - { - // Hard to ensure _loaded is set in all cases, better here to be forgiving and - // permit a clean shut-down. + _log.Information("Loading an API-key specific buffer in {Path}", subfolder); + var apiKey = _dataProtector.Unprotect(File.ReadAllText(encodedApiKeyFilePath)); - foreach (var buffer in OpenBuffers) + var buffer = new LogBuffer(subfolder, _bufferSizeBytes); + if (buffer.Peek(0).Length == 0) { - buffer.Shipper.Stop(); + _log.Information("API key-specific buffer in {Path} is empty and will be removed until more data is received", subfolder); + buffer.Dispose(); + Directory.Delete(subfolder, true); + } + else + { + var activeBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, apiKey)); + _buffersByApiKey.Add(apiKey, activeBuffer); } } + + _loaded = true; } + } - public LogBuffer GetLogBuffer(string? apiKey) + public void Start() + { + lock (_sync) { - lock (_sync) + if (!_loaded) throw new InvalidOperationException("The log buffer map is not loaded."); + + foreach (var buffer in OpenBuffers) { - if (!_loaded) throw new RequestProcessingException("The forwarder service is starting up.", HttpStatusCode.ServiceUnavailable); + buffer.Shipper.Start(); + } + } + } - if (apiKey == null) - { - if (_noApiKeyLogBuffer == null) - { - _log.Information("Creating a new default log buffer in {Path}", _bufferPath); - var buffer = new LogBuffer(_bufferPath, _bufferSizeBytes); - _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _connectionConfig.GetApiKey(_dataProtector))); - _noApiKeyLogBuffer.Shipper.Start(); - } - return _noApiKeyLogBuffer.Buffer; - } + public void Stop() + { + lock (_sync) + { + // Hard to ensure _loaded is set in all cases, better here to be forgiving and + // permit a clean shut-down. - if (_buffersByApiKey.TryGetValue(apiKey, out var existing)) - return existing.Buffer; - - var subfolder = Path.Combine(_bufferPath, Guid.NewGuid().ToString("n")); - _log.Information("Creating a new API key-specific log buffer in {Path}", subfolder); - Directory.CreateDirectory(subfolder); - File.WriteAllText(Path.Combine(subfolder, ".apikey"), _dataProtector.Protect(apiKey)); - var newBuffer = new LogBuffer(subfolder, _bufferSizeBytes); - var newActiveBuffer = new ActiveLogBuffer(newBuffer, _shipperFactory.Create(newBuffer, apiKey)); - _buffersByApiKey.Add(apiKey, newActiveBuffer); - newActiveBuffer.Shipper.Start(); - return newBuffer; + foreach (var buffer in OpenBuffers) + { + buffer.Shipper.Stop(); } } + } - public void Dispose() + public LogBuffer GetLogBuffer(string? apiKey) + { + lock (_sync) { - lock (_sync) + if (!_loaded) throw new RequestProcessingException("The forwarder service is starting up.", HttpStatusCode.ServiceUnavailable); + + if (apiKey == null) { - foreach (var buffer in OpenBuffers) + if (_noApiKeyLogBuffer == null) { - buffer.Dispose(); + _log.Information("Creating a new default log buffer in {Path}", _bufferPath); + var buffer = new LogBuffer(_bufferPath, _bufferSizeBytes); + _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _connectionConfig.GetApiKey(_dataProtector))); + _noApiKeyLogBuffer.Shipper.Start(); } + return _noApiKeyLogBuffer.Buffer; } + + if (_buffersByApiKey.TryGetValue(apiKey, out var existing)) + return existing.Buffer; + + var subfolder = Path.Combine(_bufferPath, Guid.NewGuid().ToString("n")); + _log.Information("Creating a new API key-specific log buffer in {Path}", subfolder); + Directory.CreateDirectory(subfolder); + File.WriteAllText(Path.Combine(subfolder, ".apikey"), _dataProtector.Protect(apiKey)); + var newBuffer = new LogBuffer(subfolder, _bufferSizeBytes); + var newActiveBuffer = new ActiveLogBuffer(newBuffer, _shipperFactory.Create(newBuffer, apiKey)); + _buffersByApiKey.Add(apiKey, newActiveBuffer); + newActiveBuffer.Shipper.Start(); + return newBuffer; } + } - public static void Truncate(string bufferPath) + public void Dispose() + { + lock (_sync) { - DeleteIfExists(Path.Combine(bufferPath, DataFileName)); - DeleteIfExists(Path.Combine(bufferPath, LockFileName)); - foreach (var subdirectory in Directory.GetDirectories(bufferPath)) + foreach (var buffer in OpenBuffers) { - if (File.Exists(Path.Combine(subdirectory, ApiKeyFileName))) - Directory.Delete(subdirectory, true); + buffer.Dispose(); } } + } - static void DeleteIfExists(string filePath) + public static void Truncate(string bufferPath) + { + DeleteIfExists(Path.Combine(bufferPath, DataFileName)); + DeleteIfExists(Path.Combine(bufferPath, LockFileName)); + foreach (var subdirectory in Directory.GetDirectories(bufferPath)) { - if (File.Exists(filePath)) - File.Delete(filePath); + if (File.Exists(Path.Combine(subdirectory, ApiKeyFileName))) + Directory.Delete(subdirectory, true); } + } - IEnumerable OpenBuffers + static void DeleteIfExists(string filePath) + { + if (File.Exists(filePath)) + File.Delete(filePath); + } + + IEnumerable OpenBuffers + { + get { - get - { - if (_noApiKeyLogBuffer != null) - yield return _noApiKeyLogBuffer; + if (_noApiKeyLogBuffer != null) + yield return _noApiKeyLogBuffer; - foreach (var buffer in _buffersByApiKey.Values) - yield return buffer; - } + foreach (var buffer in _buffersByApiKey.Values) + yield return buffer; } } -} +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs index d80e91df..7c95215d 100644 --- a/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs +++ b/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs @@ -18,24 +18,23 @@ using SeqCli.Forwarder.Shipper; using SeqCli.Forwarder.Storage; -namespace SeqCli.Forwarder.Multiplexing +namespace SeqCli.Forwarder.Multiplexing; + +class HttpLogShipperFactory : ILogShipperFactory { - class HttpLogShipperFactory : ILogShipperFactory - { - readonly HttpClient _outputHttpClient; - readonly ServerResponseProxy _serverResponseProxy; - readonly ConnectionConfig _outputConfig; + readonly HttpClient _outputHttpClient; + readonly ServerResponseProxy _serverResponseProxy; + readonly ConnectionConfig _outputConfig; - public HttpLogShipperFactory(ServerResponseProxy serverResponseProxy, ConnectionConfig outputConfig, HttpClient outputHttpClient) - { - _outputHttpClient = outputHttpClient; - _serverResponseProxy = serverResponseProxy ?? throw new ArgumentNullException(nameof(serverResponseProxy)); - _outputConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); - } + public HttpLogShipperFactory(ServerResponseProxy serverResponseProxy, ConnectionConfig outputConfig, HttpClient outputHttpClient) + { + _outputHttpClient = outputHttpClient; + _serverResponseProxy = serverResponseProxy ?? throw new ArgumentNullException(nameof(serverResponseProxy)); + _outputConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); + } - public LogShipper Create(LogBuffer logBuffer, string? apiKey) - { - return new HttpLogShipper(logBuffer, apiKey, _outputConfig, _serverResponseProxy, _outputHttpClient); - } + public LogShipper Create(LogBuffer logBuffer, string? apiKey) + { + return new HttpLogShipper(logBuffer, apiKey, _outputConfig, _serverResponseProxy, _outputHttpClient); } -} +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs index 85ff8df7..773f455f 100644 --- a/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs +++ b/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs @@ -15,10 +15,9 @@ using SeqCli.Forwarder.Shipper; using SeqCli.Forwarder.Storage; -namespace SeqCli.Forwarder.Multiplexing +namespace SeqCli.Forwarder.Multiplexing; + +public interface ILogShipperFactory { - public interface ILogShipperFactory - { - LogShipper Create(LogBuffer logBuffer, string? apiKey); - } -} + LogShipper Create(LogBuffer logBuffer, string? apiKey); +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs index 4d9c2b78..b6fff878 100644 --- a/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs +++ b/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs @@ -15,13 +15,12 @@ using SeqCli.Forwarder.Shipper; using SeqCli.Forwarder.Storage; -namespace SeqCli.Forwarder.Multiplexing +namespace SeqCli.Forwarder.Multiplexing; + +class InertLogShipperFactory : ILogShipperFactory { - class InertLogShipperFactory : ILogShipperFactory + public LogShipper Create(LogBuffer logBuffer, string? apiKey) { - public LogShipper Create(LogBuffer logBuffer, string? apiKey) - { - return new InertLogShipper(); - } + return new InertLogShipper(); } -} +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs b/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs index 9c629498..b52dc988 100644 --- a/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs +++ b/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs @@ -14,39 +14,38 @@ using System.Collections.Generic; -namespace SeqCli.Forwarder.Multiplexing +namespace SeqCli.Forwarder.Multiplexing; + +public class ServerResponseProxy { - public class ServerResponseProxy - { - const string EmptyResponse = "{}"; + const string EmptyResponse = "{}"; - readonly object _syncRoot = new object(); - readonly Dictionary _lastResponseByApiKey = new Dictionary(); - string _lastNoApiKeyResponse = EmptyResponse; + readonly object _syncRoot = new(); + readonly Dictionary _lastResponseByApiKey = new(); + string _lastNoApiKeyResponse = EmptyResponse; - public void SuccessResponseReturned(string? apiKey, string response) + public void SuccessResponseReturned(string? apiKey, string response) + { + lock (_syncRoot) { - lock (_syncRoot) - { - if (apiKey == null) - _lastNoApiKeyResponse = response; - else - _lastResponseByApiKey[apiKey] = response; - } + if (apiKey == null) + _lastNoApiKeyResponse = response; + else + _lastResponseByApiKey[apiKey] = response; } + } - public string GetResponseText(string? apiKey) + public string GetResponseText(string? apiKey) + { + lock (_syncRoot) { - lock (_syncRoot) - { - if (apiKey == null) - return _lastNoApiKeyResponse; + if (apiKey == null) + return _lastNoApiKeyResponse; - if (_lastResponseByApiKey.TryGetValue(apiKey, out var response)) - return response; + if (_lastResponseByApiKey.TryGetValue(apiKey, out var response)) + return response; - return EmptyResponse; - } + return EmptyResponse; } } -} +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Properties/AssemblyInfo.cs b/src/SeqCli/Forwarder/Properties/AssemblyInfo.cs deleted file mode 100644 index da681b10..00000000 --- a/src/SeqCli/Forwarder/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("Seq.Forwarder.Tests")] diff --git a/src/SeqCli/Forwarder/Schema/EventSchema.cs b/src/SeqCli/Forwarder/Schema/EventSchema.cs index 44dad50a..c13f0325 100644 --- a/src/SeqCli/Forwarder/Schema/EventSchema.cs +++ b/src/SeqCli/Forwarder/Schema/EventSchema.cs @@ -7,178 +7,177 @@ using SeqCli.Forwarder.Util; using Serilog.Parsing; -namespace SeqCli.Forwarder.Schema +namespace SeqCli.Forwarder.Schema; + +static class EventSchema { - static class EventSchema - { - static readonly MessageTemplateParser MessageTemplateParser = new MessageTemplateParser(); + static readonly MessageTemplateParser MessageTemplateParser = new(); - static readonly HashSet ClefReifiedProperties = ["@t", "@m", "@mt", "@l", "@x", "@i", "@r"]; + static readonly HashSet ClefReifiedProperties = ["@t", "@m", "@mt", "@l", "@x", "@i", "@r"]; - public static bool FromClefFormat(in int lineNumber, JObject compactFormat, [MaybeNullWhen(false)] out JObject rawFormat, [MaybeNullWhen(true)] out string error) + public static bool FromClefFormat(in int lineNumber, JObject compactFormat, [MaybeNullWhen(false)] out JObject rawFormat, [MaybeNullWhen(true)] out string error) + { + var result = new JObject(); + + var rawTimestamp = compactFormat["@t"]; + if (rawTimestamp == null) { - var result = new JObject(); + error = $"The event on line {lineNumber} does not carry an `@t` timestamp property."; + rawFormat = default; + return false; + } - var rawTimestamp = compactFormat["@t"]; - if (rawTimestamp == null) - { - error = $"The event on line {lineNumber} does not carry an `@t` timestamp property."; - rawFormat = default; - return false; - } + if (rawTimestamp.Type != JTokenType.String) + { + error = $"The event on line {lineNumber} has an invalid `@t` timestamp property; the value must be a JSON string."; + rawFormat = default; + return false; + } + + if (!DateTimeOffset.TryParse(rawTimestamp.Value(), out _)) + { + error = $"The timestamp value `{rawTimestamp}` on line {lineNumber} could not be parsed."; + rawFormat = default; + return false; + } + + result.Add("Timestamp", rawTimestamp); - if (rawTimestamp.Type != JTokenType.String) + var properties = new JObject(); + foreach (var property in compactFormat.Properties()) + { + if (property.Name.StartsWith("@@")) + properties.Add(property.Name.Substring(1), property.Value); + else if (!ClefReifiedProperties.Contains(property.Name)) + properties.Add(property.Name, property.Value); + } + + var x = compactFormat["@x"]; + if (x != null) + { + if (x.Type != JTokenType.String) { - error = $"The event on line {lineNumber} has an invalid `@t` timestamp property; the value must be a JSON string."; + error = $"The event on line {lineNumber} has a non-string `@x` exception property."; rawFormat = default; return false; } - if (!DateTimeOffset.TryParse(rawTimestamp.Value(), out _)) + result.Add("Exception", x); + } + + var l = compactFormat["@l"]; + if (l != null) + { + if (l.Type != JTokenType.String) { - error = $"The timestamp value `{rawTimestamp}` on line {lineNumber} could not be parsed."; + error = $"The event on line {lineNumber} has a non-string `@l` level property."; rawFormat = default; return false; } - result.Add("Timestamp", rawTimestamp); + result.Add("Level", l); + } - var properties = new JObject(); - foreach (var property in compactFormat.Properties()) + string? message = null; + var m = compactFormat["@m"]; + if (m != null) + { + if (m.Type != JTokenType.String) { - if (property.Name.StartsWith("@@")) - properties.Add(property.Name.Substring(1), property.Value); - else if (!ClefReifiedProperties.Contains(property.Name)) - properties.Add(property.Name, property.Value); + error = $"The event on line {lineNumber} has a non-string `@m` message property."; + rawFormat = default; + return false; } - var x = compactFormat["@x"]; - if (x != null) - { - if (x.Type != JTokenType.String) - { - error = $"The event on line {lineNumber} has a non-string `@x` exception property."; - rawFormat = default; - return false; - } - - result.Add("Exception", x); - } + message = m.Value(); + } - var l = compactFormat["@l"]; - if (l != null) + string? messageTemplate = null; + var mt = compactFormat["@mt"]; + if (mt != null) + { + if (mt.Type != JTokenType.String) { - if (l.Type != JTokenType.String) - { - error = $"The event on line {lineNumber} has a non-string `@l` level property."; - rawFormat = default; - return false; - } - - result.Add("Level", l); + error = $"The event on line {lineNumber} has a non-string `@mt` message template property."; + rawFormat = default; + return false; } - string? message = null; - var m = compactFormat["@m"]; - if (m != null) - { - if (m.Type != JTokenType.String) - { - error = $"The event on line {lineNumber} has a non-string `@m` message property."; - rawFormat = default; - return false; - } + messageTemplate = mt.Value(); + } - message = m.Value(); - } + if (message != null) + { + result.Add("RenderedMessage", message); + } + else if (messageTemplate != null && compactFormat["@r"] is JArray renderingsArray) + { + var template = MessageTemplateParser.Parse(messageTemplate); + var withFormat = template.Tokens.OfType().Where(pt => pt.Format != null); - string? messageTemplate = null; - var mt = compactFormat["@mt"]; - if (mt != null) + // ReSharper disable once PossibleMultipleEnumeration + if (withFormat.Count() == renderingsArray.Count) { - if (mt.Type != JTokenType.String) - { - error = $"The event on line {lineNumber} has a non-string `@mt` message template property."; - rawFormat = default; - return false; - } + // ReSharper disable once PossibleMultipleEnumeration + var renderingsByProperty = withFormat + .Zip(renderingsArray, (p, j) => new { p.PropertyName, Format = p.Format!, Rendering = j.Value() }) + .GroupBy(p => p.PropertyName) + .ToDictionary(g => g.Key, g => g.ToDictionaryDistinct(p => p.Format, p => p.Rendering)); - messageTemplate = mt.Value(); - } + var renderings = new JObject(); + result.Add("Renderings", renderings); - if (message != null) - { - result.Add("RenderedMessage", message); - } - else if (messageTemplate != null && compactFormat["@r"] is JArray renderingsArray) - { - var template = MessageTemplateParser.Parse(messageTemplate); - var withFormat = template.Tokens.OfType().Where(pt => pt.Format != null); - - // ReSharper disable once PossibleMultipleEnumeration - if (withFormat.Count() == renderingsArray.Count) + foreach (var (property, propertyRenderings) in renderingsByProperty) { - // ReSharper disable once PossibleMultipleEnumeration - var renderingsByProperty = withFormat - .Zip(renderingsArray, (p, j) => new { p.PropertyName, Format = p.Format!, Rendering = j.Value() }) - .GroupBy(p => p.PropertyName) - .ToDictionary(g => g.Key, g => g.ToDictionaryDistinct(p => p.Format, p => p.Rendering)); - - var renderings = new JObject(); - result.Add("Renderings", renderings); + var byFormat = new JArray(); + renderings.Add(property, byFormat); - foreach (var (property, propertyRenderings) in renderingsByProperty) + foreach (var (format, rendering) in propertyRenderings) { - var byFormat = new JArray(); - renderings.Add(property, byFormat); - - foreach (var (format, rendering) in propertyRenderings) - { - var element = new JObject {{"Format", format}, {"Rendering", rendering}}; - byFormat.Add(element); - } + var element = new JObject {{"Format", format}, {"Rendering", rendering}}; + byFormat.Add(element); } } } + } - messageTemplate ??= message ?? "No template provided"; - result.Add("MessageTemplate", messageTemplate); + messageTemplate ??= message ?? "No template provided"; + result.Add("MessageTemplate", messageTemplate); - var eventTypeToken = compactFormat["@i"]; - if (eventTypeToken != null) + var eventTypeToken = compactFormat["@i"]; + if (eventTypeToken != null) + { + if (eventTypeToken.Type == JTokenType.Integer) { - if (eventTypeToken.Type == JTokenType.Integer) - { - result.Add("EventType", uint.Parse(eventTypeToken.Value()!)); - } - else if (eventTypeToken.Type == JTokenType.String) - { - if (uint.TryParse(eventTypeToken.Value(), NumberStyles.HexNumber, + result.Add("EventType", uint.Parse(eventTypeToken.Value()!)); + } + else if (eventTypeToken.Type == JTokenType.String) + { + if (uint.TryParse(eventTypeToken.Value(), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var eventType)) - { - result.Add("EventType", eventType); - } - else - { - // Seq would calculate a hash value from the string, here. Forwarder will ignore that - // case and preserve the value in an `@i` property for now. - result.Add("@i", eventTypeToken); - } + { + result.Add("EventType", eventType); } else { - error = $"The `@i` event type value on line {lineNumber} is not in a string or numeric format."; - rawFormat = default; - return false; + // Seq would calculate a hash value from the string, here. Forwarder will ignore that + // case and preserve the value in an `@i` property for now. + result.Add("@i", eventTypeToken); } } + else + { + error = $"The `@i` event type value on line {lineNumber} is not in a string or numeric format."; + rawFormat = default; + return false; + } + } - if (properties.Count != 0) - result.Add("Properties", properties); + if (properties.Count != 0) + result.Add("Properties", properties); - rawFormat = result; - error = null; - return true; - } + rawFormat = result; + error = null; + return true; } -} +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/SeqForwarderModule.cs b/src/SeqCli/Forwarder/SeqForwarderModule.cs deleted file mode 100644 index 319c3736..00000000 --- a/src/SeqCli/Forwarder/SeqForwarderModule.cs +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Net.Http; -using System.Threading; -using Autofac; -using SeqCli.Config; -using SeqCli.Forwarder.Cryptography; -using SeqCli.Forwarder.Multiplexing; -using SeqCli.Forwarder.Web.Host; - -namespace SeqCli.Forwarder -{ - class SeqForwarderModule : Module - { - readonly string _bufferPath; - readonly SeqCliConfig _config; - - public SeqForwarderModule(string bufferPath, SeqCliConfig config) - { - _bufferPath = bufferPath ?? throw new ArgumentNullException(nameof(bufferPath)); - _config = config ?? throw new ArgumentNullException(nameof(config)); - } - - protected override void Load(ContainerBuilder builder) - { - builder.RegisterType().SingleInstance(); - builder.RegisterType() - .WithParameter("bufferPath", _bufferPath) - .SingleInstance(); - - builder.RegisterType().As(); - builder.RegisterType().SingleInstance(); - - builder.Register(c => - { - var outputConfig = c.Resolve(); - var baseUri = outputConfig.ServerUrl; - if (string.IsNullOrWhiteSpace(baseUri)) - throw new ArgumentException("The destination Seq server URL must be configured in SeqForwarder.json."); - - if (!baseUri.EndsWith("/")) - baseUri += "/"; - - // additional configuration options that require the use of SocketsHttpHandler should be added to - // this expression, using an "or" operator. - - var hasSocketHandlerOption = - (outputConfig.PooledConnectionLifetimeMilliseconds.HasValue); - - if (hasSocketHandlerOption) - { - var httpMessageHandler = new SocketsHttpHandler() - { - PooledConnectionLifetime = (outputConfig.PooledConnectionLifetimeMilliseconds.HasValue) ? TimeSpan.FromMilliseconds(outputConfig.PooledConnectionLifetimeMilliseconds.Value) : Timeout.InfiniteTimeSpan, - }; - - return new HttpClient(httpMessageHandler) { BaseAddress = new Uri(baseUri) }; - } - - return new HttpClient() { BaseAddress = new Uri(baseUri) }; - - }).SingleInstance(); - - builder.RegisterInstance(StringDataProtector.CreatePlatformDefault()); - - builder.RegisterInstance(_config); - builder.RegisterInstance(_config.Forwarder.Api); - builder.RegisterInstance(_config.Forwarder.Diagnostics); - builder.RegisterInstance(_config.Forwarder.Storage); - } - } -} diff --git a/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs b/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs index 96c9c7a4..9439f4ba 100644 --- a/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs +++ b/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs @@ -14,61 +14,60 @@ using System; -namespace SeqCli.Forwarder.Shipper +namespace SeqCli.Forwarder.Shipper; + +class ExponentialBackoffConnectionSchedule { - class ExponentialBackoffConnectionSchedule - { - static readonly TimeSpan MinimumBackoffPeriod = TimeSpan.FromSeconds(5); - static readonly TimeSpan MaximumBackoffInterval = TimeSpan.FromMinutes(10); + static readonly TimeSpan MinimumBackoffPeriod = TimeSpan.FromSeconds(5); + static readonly TimeSpan MaximumBackoffInterval = TimeSpan.FromMinutes(10); - readonly TimeSpan _period; + readonly TimeSpan _period; - int _failuresSinceSuccessfulConnection; + int _failuresSinceSuccessfulConnection; - public ExponentialBackoffConnectionSchedule(TimeSpan period) - { - if (period < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(period), "The connection retry period must be a positive timespan"); + public ExponentialBackoffConnectionSchedule(TimeSpan period) + { + if (period < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(period), "The connection retry period must be a positive timespan."); - _period = period; - } + _period = period; + } - public void MarkSuccess() - { - _failuresSinceSuccessfulConnection = 0; - } + public void MarkSuccess() + { + _failuresSinceSuccessfulConnection = 0; + } - public void MarkFailure() - { - ++_failuresSinceSuccessfulConnection; - } + public void MarkFailure() + { + ++_failuresSinceSuccessfulConnection; + } - public bool LastConnectionFailed => _failuresSinceSuccessfulConnection != 0; + public bool LastConnectionFailed => _failuresSinceSuccessfulConnection != 0; - public TimeSpan NextInterval + public TimeSpan NextInterval + { + get { - get - { - // Available, and first failure, just try the batch interval - if (_failuresSinceSuccessfulConnection <= 1) return _period; + // Available, and first failure, just try the batch interval + if (_failuresSinceSuccessfulConnection <= 1) return _period; - // Second failure, start ramping up the interval - first 2x, then 4x, ... - var backoffFactor = Math.Pow(2, (_failuresSinceSuccessfulConnection - 1)); + // Second failure, start ramping up the interval - first 2x, then 4x, ... + var backoffFactor = Math.Pow(2, (_failuresSinceSuccessfulConnection - 1)); - // If the period is ridiculously short, give it a boost so we get some - // visible backoff. - var backoffPeriod = Math.Max(_period.Ticks, MinimumBackoffPeriod.Ticks); + // If the period is ridiculously short, give it a boost so we get some + // visible backoff. + var backoffPeriod = Math.Max(_period.Ticks, MinimumBackoffPeriod.Ticks); - // The "ideal" interval - var backedOff = (long)(backoffPeriod * backoffFactor); + // The "ideal" interval + var backedOff = (long)(backoffPeriod * backoffFactor); - // Capped to the maximum interval - var cappedBackoff = Math.Min(MaximumBackoffInterval.Ticks, backedOff); + // Capped to the maximum interval + var cappedBackoff = Math.Min(MaximumBackoffInterval.Ticks, backedOff); - // Unless that's shorter than the base interval, in which case we'll just apply the period - var actual = Math.Max(_period.Ticks, cappedBackoff); + // Unless that's shorter than the base interval, in which case we'll just apply the period + var actual = Math.Max(_period.Ticks, cappedBackoff); - return TimeSpan.FromTicks(actual); - } + return TimeSpan.FromTicks(actual); } } -} +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs b/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs index 41354e6d..f39159b1 100644 --- a/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs +++ b/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs @@ -26,226 +26,225 @@ using SeqCli.Forwarder.Util; using Serilog; -namespace SeqCli.Forwarder.Shipper +namespace SeqCli.Forwarder.Shipper; + +sealed class HttpLogShipper : LogShipper { - sealed class HttpLogShipper : LogShipper - { - const string BulkUploadResource = "api/events/raw"; + const string BulkUploadResource = "api/events/raw"; - readonly string? _apiKey; - readonly LogBuffer _logBuffer; - readonly ConnectionConfig _outputConfig; - readonly HttpClient _httpClient; - readonly ExponentialBackoffConnectionSchedule _connectionSchedule; - readonly ServerResponseProxy _serverResponseProxy; - DateTime _nextRequiredLevelCheck; + readonly string? _apiKey; + readonly LogBuffer _logBuffer; + readonly ConnectionConfig _outputConfig; + readonly HttpClient _httpClient; + readonly ExponentialBackoffConnectionSchedule _connectionSchedule; + readonly ServerResponseProxy _serverResponseProxy; + DateTime _nextRequiredLevelCheck; - readonly object _stateLock = new object(); - readonly Timer _timer; - bool _started; + readonly object _stateLock = new(); + readonly Timer _timer; + bool _started; - volatile bool _unloading; + volatile bool _unloading; - static readonly TimeSpan QuietWaitPeriod = TimeSpan.FromSeconds(2), MaximumConnectionInterval = TimeSpan.FromMinutes(2); + static readonly TimeSpan QuietWaitPeriod = TimeSpan.FromSeconds(2), MaximumConnectionInterval = TimeSpan.FromMinutes(2); - public HttpLogShipper(LogBuffer logBuffer, string? apiKey, ConnectionConfig outputConfig, ServerResponseProxy serverResponseProxy, HttpClient outputHttpClient) - { - _apiKey = apiKey; - _httpClient = outputHttpClient ?? throw new ArgumentNullException(nameof(outputHttpClient)); - _logBuffer = logBuffer ?? throw new ArgumentNullException(nameof(logBuffer)); - _outputConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); - _serverResponseProxy = serverResponseProxy ?? throw new ArgumentNullException(nameof(serverResponseProxy)); - _connectionSchedule = new ExponentialBackoffConnectionSchedule(QuietWaitPeriod); - _timer = new Timer(s => OnTick()); - } + public HttpLogShipper(LogBuffer logBuffer, string? apiKey, ConnectionConfig outputConfig, ServerResponseProxy serverResponseProxy, HttpClient outputHttpClient) + { + _apiKey = apiKey; + _httpClient = outputHttpClient ?? throw new ArgumentNullException(nameof(outputHttpClient)); + _logBuffer = logBuffer ?? throw new ArgumentNullException(nameof(logBuffer)); + _outputConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); + _serverResponseProxy = serverResponseProxy ?? throw new ArgumentNullException(nameof(serverResponseProxy)); + _connectionSchedule = new ExponentialBackoffConnectionSchedule(QuietWaitPeriod); + _timer = new Timer(_ => OnTick()); + } - public override void Start() + public override void Start() + { + lock (_stateLock) { - lock (_stateLock) - { - if (_started) - throw new InvalidOperationException("The shipper has already started."); + if (_started) + throw new InvalidOperationException("The shipper has already started."); - if (_unloading) - throw new InvalidOperationException("The shipper is unloading."); + if (_unloading) + throw new InvalidOperationException("The shipper is unloading."); - Log.Information("Log shipper started, events will be dispatched to {ServerUrl}", _outputConfig.ServerUrl); + Log.Information("Log shipper started, events will be dispatched to {ServerUrl}", _outputConfig.ServerUrl); - _nextRequiredLevelCheck = DateTime.UtcNow.Add(MaximumConnectionInterval); - _started = true; - SetTimer(); - } + _nextRequiredLevelCheck = DateTime.UtcNow.Add(MaximumConnectionInterval); + _started = true; + SetTimer(); } + } - public override void Stop() + public override void Stop() + { + lock (_stateLock) { - lock (_stateLock) - { - if (_unloading) - return; + if (_unloading) + return; - _unloading = true; + _unloading = true; - if (!_started) - return; - } - - var wh = new ManualResetEvent(false); - if (_timer.Dispose(wh)) - wh.WaitOne(); + if (!_started) + return; } + + var wh = new ManualResetEvent(false); + if (_timer.Dispose(wh)) + wh.WaitOne(); + } - public override void Dispose() - { - Stop(); - } + public override void Dispose() + { + Stop(); + } - void SetTimer() - { - _timer.Change(_connectionSchedule.NextInterval, Timeout.InfiniteTimeSpan); - } + void SetTimer() + { + _timer.Change(_connectionSchedule.NextInterval, Timeout.InfiniteTimeSpan); + } - void OnTick() - { - OnTickAsync().Wait(); - } + void OnTick() + { + OnTickAsync().Wait(); + } - async Task OnTickAsync() + async Task OnTickAsync() + { + try { - try + var sendingSingles = 0; + do { - var sendingSingles = 0; - do + var available = _logBuffer.Peek((int)_outputConfig.PayloadLimitBytes); + if (available.Length == 0) { - var available = _logBuffer.Peek((int)_outputConfig.PayloadLimitBytes); - if (available.Length == 0) + if (DateTime.UtcNow < _nextRequiredLevelCheck || _connectionSchedule.LastConnectionFailed) { - if (DateTime.UtcNow < _nextRequiredLevelCheck || _connectionSchedule.LastConnectionFailed) - { - // For whatever reason, there's nothing waiting to send. This means we should try connecting again at the - // regular interval, so mark the attempt as successful. - _connectionSchedule.MarkSuccess(); - break; - } + // For whatever reason, there's nothing waiting to send. This means we should try connecting again at the + // regular interval, so mark the attempt as successful. + _connectionSchedule.MarkSuccess(); + break; } + } - MakePayload(available, sendingSingles > 0, out Stream payload, out ulong lastIncluded); + MakePayload(available, sendingSingles > 0, out Stream payload, out ulong lastIncluded); - var content = new StreamContent(new UnclosableStreamWrapper(payload)); - content.Headers.ContentType = new MediaTypeHeaderValue("application/json") - { - CharSet = Encoding.UTF8.WebName - }; + var content = new StreamContent(new UnclosableStreamWrapper(payload)); + content.Headers.ContentType = new MediaTypeHeaderValue("application/json") + { + CharSet = Encoding.UTF8.WebName + }; - if (_apiKey != null) - { - content.Headers.Add(SeqApi.ApiKeyHeaderName, _apiKey); - } + if (_apiKey != null) + { + content.Headers.Add(SeqApi.ApiKeyHeaderName, _apiKey); + } - var result = await _httpClient.PostAsync(BulkUploadResource, content); - if (result.IsSuccessStatusCode) - { - _connectionSchedule.MarkSuccess(); - _logBuffer.Dequeue(lastIncluded); - if (sendingSingles > 0) - sendingSingles--; + var result = await _httpClient.PostAsync(BulkUploadResource, content); + if (result.IsSuccessStatusCode) + { + _connectionSchedule.MarkSuccess(); + _logBuffer.Dequeue(lastIncluded); + if (sendingSingles > 0) + sendingSingles--; - _serverResponseProxy.SuccessResponseReturned(_apiKey, await result.Content.ReadAsStringAsync()); - _nextRequiredLevelCheck = DateTime.UtcNow.Add(MaximumConnectionInterval); - } - else if (result.StatusCode == HttpStatusCode.BadRequest || - result.StatusCode == HttpStatusCode.RequestEntityTooLarge) - { - // The connection attempt was successful - the payload we sent was the problem. - _connectionSchedule.MarkSuccess(); + _serverResponseProxy.SuccessResponseReturned(_apiKey, await result.Content.ReadAsStringAsync()); + _nextRequiredLevelCheck = DateTime.UtcNow.Add(MaximumConnectionInterval); + } + else if (result.StatusCode == HttpStatusCode.BadRequest || + result.StatusCode == HttpStatusCode.RequestEntityTooLarge) + { + // The connection attempt was successful - the payload we sent was the problem. + _connectionSchedule.MarkSuccess(); - if (sendingSingles != 0) - { - payload.Position = 0; - var payloadText = await new StreamReader(payload, Encoding.UTF8).ReadToEndAsync(); - Log.Error("HTTP shipping failed with {StatusCode}: {Result}; payload was {InvalidPayload}", result.StatusCode, await result.Content.ReadAsStringAsync(), payloadText); - _logBuffer.Dequeue(lastIncluded); - sendingSingles = 0; - } - else - { - // Unscientific (should "binary search" in batches) but sending the next - // hundred events singly should flush out the problematic one. - sendingSingles = 100; - } + if (sendingSingles != 0) + { + payload.Position = 0; + var payloadText = await new StreamReader(payload, Encoding.UTF8).ReadToEndAsync(); + Log.Error("HTTP shipping failed with {StatusCode}: {Result}; payload was {InvalidPayload}", result.StatusCode, await result.Content.ReadAsStringAsync(), payloadText); + _logBuffer.Dequeue(lastIncluded); + sendingSingles = 0; } else { - _connectionSchedule.MarkFailure(); - Log.Error("Received failed HTTP shipping result {StatusCode}: {Result}", result.StatusCode, await result.Content.ReadAsStringAsync()); - break; + // Unscientific (should "binary search" in batches) but sending the next + // hundred events singly should flush out the problematic one. + sendingSingles = 100; } } - while (true); - } - catch (HttpRequestException hex) - { - Log.Warning(hex, "HTTP request failed when sending a batch from the log shipper"); - _connectionSchedule.MarkFailure(); - } - catch (Exception ex) - { - Log.Error(ex, "Exception while sending a batch from the log shipper"); - _connectionSchedule.MarkFailure(); - } - finally - { - lock (_stateLock) + else { - if (!_unloading) - SetTimer(); + _connectionSchedule.MarkFailure(); + Log.Error("Received failed HTTP shipping result {StatusCode}: {Result}", result.StatusCode, await result.Content.ReadAsStringAsync()); + break; } } + while (true); } - - void MakePayload(LogBufferEntry[] entries, bool oneOnly, out Stream utf8Payload, out ulong lastIncluded) + catch (HttpRequestException hex) { - if (entries == null) throw new ArgumentNullException(nameof(entries)); - lastIncluded = 0; - - var raw = new MemoryStream(); - var content = new StreamWriter(raw, Encoding.UTF8); - content.Write("{\"Events\":["); - content.Flush(); - var contentRemainingBytes = (int) _outputConfig.PayloadLimitBytes - 13; // Includes closing delims - - var delimStart = ""; - foreach (var logBufferEntry in entries) + Log.Warning(hex, "HTTP request failed when sending a batch from the log shipper"); + _connectionSchedule.MarkFailure(); + } + catch (Exception ex) + { + Log.Error(ex, "Exception while sending a batch from the log shipper"); + _connectionSchedule.MarkFailure(); + } + finally + { + lock (_stateLock) { - if ((ulong)logBufferEntry.Value.Length > _outputConfig.EventBodyLimitBytes) - { - Log.Information("Oversized event will be skipped, {Payload}", Encoding.UTF8.GetString(logBufferEntry.Value)); - lastIncluded = logBufferEntry.Key; - continue; - } - - // lastIncluded indicates we've added at least one event - if (lastIncluded != 0 && contentRemainingBytes - (delimStart.Length + logBufferEntry.Value.Length) < 0) - break; + if (!_unloading) + SetTimer(); + } + } + } - content.Write(delimStart); - content.Flush(); - contentRemainingBytes -= delimStart.Length; + void MakePayload(LogBufferEntry[] entries, bool oneOnly, out Stream utf8Payload, out ulong lastIncluded) + { + if (entries == null) throw new ArgumentNullException(nameof(entries)); + lastIncluded = 0; - raw.Write(logBufferEntry.Value, 0, logBufferEntry.Value.Length); - contentRemainingBytes -= logBufferEntry.Value.Length; + var raw = new MemoryStream(); + var content = new StreamWriter(raw, Encoding.UTF8); + content.Write("{\"Events\":["); + content.Flush(); + var contentRemainingBytes = (int) _outputConfig.PayloadLimitBytes - 13; // Includes closing delimiters + var delimStart = ""; + foreach (var logBufferEntry in entries) + { + if ((ulong)logBufferEntry.Value.Length > _outputConfig.EventBodyLimitBytes) + { + Log.Information("Oversize event will be skipped, {Payload}", Encoding.UTF8.GetString(logBufferEntry.Value)); lastIncluded = logBufferEntry.Key; - - delimStart = ","; - if (oneOnly) - break; + continue; } - content.Write("]}"); + // lastIncluded indicates we've added at least one event + if (lastIncluded != 0 && contentRemainingBytes - (delimStart.Length + logBufferEntry.Value.Length) < 0) + break; + + content.Write(delimStart); content.Flush(); - raw.Position = 0; - utf8Payload = raw; + contentRemainingBytes -= delimStart.Length; + + raw.Write(logBufferEntry.Value, 0, logBufferEntry.Value.Length); + contentRemainingBytes -= logBufferEntry.Value.Length; + + lastIncluded = logBufferEntry.Key; + + delimStart = ","; + if (oneOnly) + break; } + + content.Write("]}"); + content.Flush(); + raw.Position = 0; + utf8Payload = raw; } -} +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs b/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs index 4288b670..1ae106e3 100644 --- a/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs +++ b/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs @@ -12,20 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -namespace SeqCli.Forwarder.Shipper +namespace SeqCli.Forwarder.Shipper; + +class InertLogShipper : LogShipper { - class InertLogShipper : LogShipper - { - public override void Start() - { - } + public override void Start() + { + } - public override void Stop() - { - } + public override void Stop() + { + } - public override void Dispose() - { - } + public override void Dispose() + { } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Shipper/LogShipper.cs b/src/SeqCli/Forwarder/Shipper/LogShipper.cs index 82594955..83e8beb3 100644 --- a/src/SeqCli/Forwarder/Shipper/LogShipper.cs +++ b/src/SeqCli/Forwarder/Shipper/LogShipper.cs @@ -14,12 +14,11 @@ using System; -namespace SeqCli.Forwarder.Shipper +namespace SeqCli.Forwarder.Shipper; + +public abstract class LogShipper : IDisposable { - public abstract class LogShipper : IDisposable - { - public abstract void Start(); - public abstract void Stop(); - public abstract void Dispose(); - } -} + public abstract void Start(); + public abstract void Stop(); + public abstract void Dispose(); +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Shipper/SeqApi.cs b/src/SeqCli/Forwarder/Shipper/SeqApi.cs index 83333823..5e7c45e2 100644 --- a/src/SeqCli/Forwarder/Shipper/SeqApi.cs +++ b/src/SeqCli/Forwarder/Shipper/SeqApi.cs @@ -12,10 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -namespace SeqCli.Forwarder.Shipper +namespace SeqCli.Forwarder.Shipper; + +static class SeqApi { - static class SeqApi - { - public const string ApiKeyHeaderName = "X-Seq-ApiKey"; - } + public const string ApiKeyHeaderName = "X-Seq-ApiKey"; } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Storage/LogBuffer.cs b/src/SeqCli/Forwarder/Storage/LogBuffer.cs index 5983df2c..f58fe8db 100644 --- a/src/SeqCli/Forwarder/Storage/LogBuffer.cs +++ b/src/SeqCli/Forwarder/Storage/LogBuffer.cs @@ -16,265 +16,264 @@ using System.Collections.Generic; using Serilog; -namespace SeqCli.Forwarder.Storage +namespace SeqCli.Forwarder.Storage; + +public class LogBuffer : IDisposable { - public class LogBuffer : IDisposable + readonly ulong _bufferSizeBytes; + // readonly LightningEnvironment _env; + readonly object _sync = new(); + bool _isDisposed; + ulong _nextId = 0, _entries = 0, _writtenSinceRotateCheck; + + public LogBuffer(string bufferPath, ulong bufferSizeBytes) { - readonly ulong _bufferSizeBytes; - // readonly LightningEnvironment _env; - readonly object _sync = new object(); - bool _isDisposed; - ulong _nextId = 0, _entries = 0, _writtenSinceRotateCheck; + _bufferSizeBytes = bufferSizeBytes; + if (bufferPath == null) throw new ArgumentNullException(nameof(bufferPath)); + + // _env = new LightningEnvironment(bufferPath) + // { + // // Sparse; we'd hope fragmentation never gets this bad... + // MapSize = (long) bufferSizeBytes*10 + // }; + // + // _env.Open(); + // + // using (var tx = _env.BeginTransaction()) + // using (var db = tx.OpenDatabase()) + // { + // using (var cur = tx.CreateCursor(db)) + // { + // if (!cur.MoveToLast()) + // { + // _nextId = 1; + // } + // else + // { + // var current = cur.GetCurrent(); + // _nextId = ByteKeyToULongKey(current.Key) + 1; + // _entries = (ulong) tx.GetEntriesCount(db); + // } + // } + // } + + Log.Information("Log buffer open on {BufferPath}; {Entries} entries, next key will be {NextId}", bufferPath, _entries, _nextId); + } - public LogBuffer(string bufferPath, ulong bufferSizeBytes) + public void Dispose() + { + lock (_sync) { - _bufferSizeBytes = bufferSizeBytes; - if (bufferPath == null) throw new ArgumentNullException(nameof(bufferPath)); + if (!_isDisposed) + { + _isDisposed = true; + // _env.Dispose(); + } + } + } - // _env = new LightningEnvironment(bufferPath) - // { - // // Sparse; we'd hope fragmentation never gets this bad... - // MapSize = (long) bufferSizeBytes*10 - // }; - // - // _env.Open(); + public void Enqueue(byte[][] values) + { + if (values == null) throw new ArgumentNullException(nameof(values)); + + lock (_sync) + { + RequireNotDisposed(); + + // var totalPayloadWritten = 0UL; // // using (var tx = _env.BeginTransaction()) // using (var db = tx.OpenDatabase()) // { - // using (var cur = tx.CreateCursor(db)) + // foreach (var v in values) // { - // if (!cur.MoveToLast()) - // { - // _nextId = 1; - // } - // else - // { - // var current = cur.GetCurrent(); - // _nextId = ByteKeyToULongKey(current.Key) + 1; - // _entries = (ulong) tx.GetEntriesCount(db); - // } + // if (v == null) throw new ArgumentException("Value array may not contain null."); + // + // tx.Put(db, ULongKeyToByteKey(_nextId++), v); + // totalPayloadWritten += (ulong) v.Length; // } + // + // tx.Commit(); + // _entries += (ulong) values.Length; + // _writtenSinceRotateCheck += totalPayloadWritten; // } - - Log.Information("Log buffer open on {BufferPath}; {Entries} entries, next key will be {NextId}", bufferPath, _entries, _nextId); - } - public void Dispose() - { - lock (_sync) - { - if (!_isDisposed) - { - _isDisposed = true; - // _env.Dispose(); - } - } + RotateIfRequired(); } + } - public void Enqueue(byte[][] values) - { - if (values == null) throw new ArgumentNullException(nameof(values)); - - lock (_sync) - { - RequireNotDisposed(); - - // var totalPayloadWritten = 0UL; - // - // using (var tx = _env.BeginTransaction()) - // using (var db = tx.OpenDatabase()) - // { - // foreach (var v in values) - // { - // if (v == null) throw new ArgumentException("Value array may not contain null."); - // - // tx.Put(db, ULongKeyToByteKey(_nextId++), v); - // totalPayloadWritten += (ulong) v.Length; - // } - // - // tx.Commit(); - // _entries += (ulong) values.Length; - // _writtenSinceRotateCheck += totalPayloadWritten; - // } + void RotateIfRequired() + { + if (_writtenSinceRotateCheck < _bufferSizeBytes/10) + return; - RotateIfRequired(); - } - } + _writtenSinceRotateCheck = 0; + // + // using (var tx = _env.BeginTransaction()) + // using (var db = tx.OpenDatabase()) + // { + // int err; + // if (0 != (err = Lmdb.mdb_env_info(_env.Handle(), out var estat))) + // throw new Exception(Lmdb.mdb_strerror(err)); + // + // MDBStat stat; + // if (0 != (err = Lmdb.mdb_stat(tx.Handle(), db.Handle(), out stat))) + // throw new Exception(Lmdb.mdb_strerror(err)); + // + // // http://www.openldap.org/lists/openldap-technical/201303/msg00145.html + // // 1) MDB_stat gives you the page size. + // // 2) MDB_envinfo tells the mapsize and the last_pgno.If you divide mapsize + // // by pagesize you'll get max pgno. The MAP_FULL error is returned when last_pgno reaches max pgno. + // + // var targetPages = _bufferSizeBytes/stat.ms_psize; + // if ((ulong) estat.me_last_pgno < targetPages && (double) (ulong) estat.me_last_pgno/targetPages < 0.75) + // return; + // + // var count = tx.GetEntriesCount(db); + // if (count == 0) + // { + // Log.Warning("Attempting to rotate buffer but no events are present"); + // return; + // } + // + // var toPurge = Math.Max(count / 4, 1); + // Log.Warning("Buffer is full; dropping {ToPurge} events to make room for new ones", + // toPurge); + // + // using (var cur = tx.CreateCursor(db)) + // { + // cur.MoveToFirst(); + // + // for (var i = 0; i < toPurge; ++i) + // { + // cur.Delete(); + // cur.MoveNext(); + // } + // } + // + // tx.Commit(); + // } + } - void RotateIfRequired() + public LogBufferEntry[] Peek(int maxValueBytesHint) + { + lock (_sync) { - if (_writtenSinceRotateCheck < _bufferSizeBytes/10) - return; + RequireNotDisposed(); - _writtenSinceRotateCheck = 0; + var entries = new List(); // - // using (var tx = _env.BeginTransaction()) + // using (var tx = _env.BeginTransaction(TransactionBeginFlags.ReadOnly)) // using (var db = tx.OpenDatabase()) // { - // int err; - // if (0 != (err = Lmdb.mdb_env_info(_env.Handle(), out var estat))) - // throw new Exception(Lmdb.mdb_strerror(err)); + // using (var cur = tx.CreateCursor(db)) + // { + // if (cur.MoveToFirst()) + // { + // var entriesBytes = 0; // - // MDBStat stat; - // if (0 != (err = Lmdb.mdb_stat(tx.Handle(), db.Handle(), out stat))) - // throw new Exception(Lmdb.mdb_strerror(err)); + // do + // { + // var current = cur.GetCurrent(); + // var entry = new LogBufferEntry + // { + // Key = ByteKeyToULongKey(current.Key), + // Value = current.Value + // }; // - // // http://www.openldap.org/lists/openldap-technical/201303/msg00145.html - // // 1) MDB_stat gives you the page size. - // // 2) MDB_envinfo tells the mapsize and the last_pgno.If you divide mapsize - // // by pagesize you'll get max pgno. The MAP_FULL error is returned when last_pgno reaches max pgno. + // entriesBytes += entry.Value.Length; + // if (entries.Count != 0 && entriesBytes > maxValueBytesHint) + // break; // - // var targetPages = _bufferSizeBytes/stat.ms_psize; - // if ((ulong) estat.me_last_pgno < targetPages && (double) (ulong) estat.me_last_pgno/targetPages < 0.75) - // return; + // entries.Add(entry); // - // var count = tx.GetEntriesCount(db); - // if (count == 0) - // { - // Log.Warning("Attempting to rotate buffer but no events are present"); - // return; + // } while (cur.MoveNext()); + // } // } + // } + + return entries.ToArray(); + } + } + + public void Dequeue(ulong toKey) + { + lock (_sync) + { + RequireNotDisposed(); + + // ulong deleted = 0; // - // var toPurge = Math.Max(count / 4, 1); - // Log.Warning("Buffer is full; dropping {ToPurge} events to make room for new ones", - // toPurge); - // + // using (var tx = _env.BeginTransaction()) + // using (var db = tx.OpenDatabase()) + // { // using (var cur = tx.CreateCursor(db)) // { - // cur.MoveToFirst(); - // - // for (var i = 0; i < toPurge; ++i) + // if (cur.MoveToFirst()) // { - // cur.Delete(); - // cur.MoveNext(); + // do + // { + // var current = cur.GetCurrent(); + // if (ByteKeyToULongKey(current.Key) > toKey) + // break; + // + // cur.Delete(); + // deleted++; + // } while (cur.MoveNext()); // } // } // // tx.Commit(); + // _entries -= deleted; // } } + } - public LogBufferEntry[] Peek(int maxValueBytesHint) - { - lock (_sync) - { - RequireNotDisposed(); - - var entries = new List(); - // - // using (var tx = _env.BeginTransaction(TransactionBeginFlags.ReadOnly)) - // using (var db = tx.OpenDatabase()) - // { - // using (var cur = tx.CreateCursor(db)) - // { - // if (cur.MoveToFirst()) - // { - // var entriesBytes = 0; - // - // do - // { - // var current = cur.GetCurrent(); - // var entry = new LogBufferEntry - // { - // Key = ByteKeyToULongKey(current.Key), - // Value = current.Value - // }; - // - // entriesBytes += entry.Value.Length; - // if (entries.Count != 0 && entriesBytes > maxValueBytesHint) - // break; - // - // entries.Add(entry); - // - // } while (cur.MoveNext()); - // } - // } - // } - - return entries.ToArray(); - } - } - - public void Dequeue(ulong toKey) - { - lock (_sync) - { - RequireNotDisposed(); - - // ulong deleted = 0; - // - // using (var tx = _env.BeginTransaction()) - // using (var db = tx.OpenDatabase()) - // { - // using (var cur = tx.CreateCursor(db)) - // { - // if (cur.MoveToFirst()) - // { - // do - // { - // var current = cur.GetCurrent(); - // if (ByteKeyToULongKey(current.Key) > toKey) - // break; - // - // cur.Delete(); - // deleted++; - // } while (cur.MoveNext()); - // } - // } - // - // tx.Commit(); - // _entries -= deleted; - // } - } - } + void RequireNotDisposed() + { + if (_isDisposed) + throw new ObjectDisposedException(typeof(LogBuffer).FullName); + } - void RequireNotDisposed() - { - if (_isDisposed) - throw new ObjectDisposedException(typeof(LogBuffer).FullName); - } + static ulong ByteKeyToULongKey(byte[] key) + { + var copy = new byte[key.Length]; + for (var i = 0; i < key.Length; ++i) + copy[copy.Length - (i + 1)] = key[i]; - static ulong ByteKeyToULongKey(byte[] key) - { - var copy = new byte[key.Length]; - for (var i = 0; i < key.Length; ++i) - copy[copy.Length - (i + 1)] = key[i]; + return BitConverter.ToUInt64(copy, 0); + } - return BitConverter.ToUInt64(copy, 0); - } + static byte[] ULongKeyToByteKey(ulong key) + { + var k = BitConverter.GetBytes(key); + Array.Reverse(k); + return k; + } - static byte[] ULongKeyToByteKey(ulong key) - { - var k = BitConverter.GetBytes(key); - Array.Reverse(k); - return k; - } + public void Enumerate(Action action) + { + if (action == null) throw new ArgumentNullException(nameof(action)); - public void Enumerate(Action action) + lock (_sync) { - if (action == null) throw new ArgumentNullException(nameof(action)); - - lock (_sync) - { - RequireNotDisposed(); + RequireNotDisposed(); - // using (var tx = _env.BeginTransaction(TransactionBeginFlags.ReadOnly)) - // using (var db = tx.OpenDatabase()) - // { - // using (var cur = tx.CreateCursor(db)) - // { - // if (cur.MoveToFirst()) - // { - // do - // { - // var current = cur.GetCurrent(); - // action(ByteKeyToULongKey(current.Key), current.Value); - // } while (cur.MoveNext()); - // } - // } - // } - } + // using (var tx = _env.BeginTransaction(TransactionBeginFlags.ReadOnly)) + // using (var db = tx.OpenDatabase()) + // { + // using (var cur = tx.CreateCursor(db)) + // { + // if (cur.MoveToFirst()) + // { + // do + // { + // var current = cur.GetCurrent(); + // action(ByteKeyToULongKey(current.Key), current.Value); + // } while (cur.MoveNext()); + // } + // } + // } } } -} +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs b/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs index fb0851a2..649be980 100644 --- a/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs +++ b/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs @@ -14,11 +14,10 @@ // ReSharper disable InconsistentNaming -namespace SeqCli.Forwarder.Storage +namespace SeqCli.Forwarder.Storage; + +public struct LogBufferEntry { - public struct LogBufferEntry - { - public ulong Key; - public byte[] Value; - } + public ulong Key; + public byte[] Value; } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Util/CaptiveProcess.cs b/src/SeqCli/Forwarder/Util/CaptiveProcess.cs index eea47c83..0c2e3e44 100644 --- a/src/SeqCli/Forwarder/Util/CaptiveProcess.cs +++ b/src/SeqCli/Forwarder/Util/CaptiveProcess.cs @@ -16,67 +16,66 @@ using System.Diagnostics; using System.Threading; -namespace SeqCli.Forwarder.Util +namespace SeqCli.Forwarder.Util; + +public static class CaptiveProcess { - public static class CaptiveProcess + public static int Run( + string fullExePath, + string? args = null, + Action? writeStdout = null, + Action? writeStderr = null, + string? workingDirectory = null) { - public static int Run( - string fullExePath, - string? args = null, - Action? writeStdout = null, - Action? writeStderr = null, - string? workingDirectory = null) - { - if (fullExePath == null) throw new ArgumentNullException(nameof(fullExePath)); + if (fullExePath == null) throw new ArgumentNullException(nameof(fullExePath)); - args ??= ""; - writeStdout ??= delegate { }; - writeStderr ??= delegate { }; + args ??= ""; + writeStdout ??= delegate { }; + writeStderr ??= delegate { }; - var startInfo = new ProcessStartInfo - { - UseShellExecute = false, - RedirectStandardError = true, - RedirectStandardOutput = true, - WindowStyle = ProcessWindowStyle.Hidden, - CreateNoWindow = true, - ErrorDialog = false, - FileName = fullExePath, - Arguments = args - }; + var startInfo = new ProcessStartInfo + { + UseShellExecute = false, + RedirectStandardError = true, + RedirectStandardOutput = true, + WindowStyle = ProcessWindowStyle.Hidden, + CreateNoWindow = true, + ErrorDialog = false, + FileName = fullExePath, + Arguments = args + }; - if (!string.IsNullOrEmpty(workingDirectory)) - startInfo.WorkingDirectory = workingDirectory; + if (!string.IsNullOrEmpty(workingDirectory)) + startInfo.WorkingDirectory = workingDirectory; - using var process = Process.Start(startInfo)!; - using var outputComplete = new ManualResetEvent(false); - using var errorComplete = new ManualResetEvent(false); - // ReSharper disable AccessToDisposedClosure + using var process = Process.Start(startInfo)!; + using var outputComplete = new ManualResetEvent(false); + using var errorComplete = new ManualResetEvent(false); + // ReSharper disable AccessToDisposedClosure - process.OutputDataReceived += (_, e) => - { - if (e.Data == null) - outputComplete.Set(); - else - writeStdout(e.Data); - }; - process.BeginOutputReadLine(); + process.OutputDataReceived += (_, e) => + { + if (e.Data == null) + outputComplete.Set(); + else + writeStdout(e.Data); + }; + process.BeginOutputReadLine(); - process.ErrorDataReceived += (_, e) => - { - if (e.Data == null) - errorComplete.Set(); - else - writeStderr(e.Data); - }; - process.BeginErrorReadLine(); + process.ErrorDataReceived += (_, e) => + { + if (e.Data == null) + errorComplete.Set(); + else + writeStderr(e.Data); + }; + process.BeginErrorReadLine(); - process.WaitForExit(); + process.WaitForExit(); - outputComplete.WaitOne(); - errorComplete.WaitOne(); + outputComplete.WaitOne(); + errorComplete.WaitOne(); - return process.ExitCode; - } + return process.ExitCode; } -} +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs b/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs index da930090..6f78a435 100644 --- a/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs +++ b/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs @@ -1,20 +1,19 @@ using System; using System.Collections.Generic; -namespace SeqCli.Forwarder.Util +namespace SeqCli.Forwarder.Util; + +static class EnumerableExtensions { - static class EnumerableExtensions - { - public static Dictionary ToDictionaryDistinct( - this IEnumerable enumerable, Func keySelector, Func valueSelector) + public static Dictionary ToDictionaryDistinct( + this IEnumerable enumerable, Func keySelector, Func valueSelector) where TKey: notnull + { + var result = new Dictionary(); + foreach (var e in enumerable) { - var result = new Dictionary(); - foreach (var e in enumerable) - { - result[keySelector(e)] = valueSelector(e); - } - return result; + result[keySelector(e)] = valueSelector(e); } + return result; } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs b/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs index 26535bbf..e0449984 100644 --- a/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs +++ b/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs @@ -1,20 +1,19 @@ -namespace SeqCli.Forwarder.Util +namespace SeqCli.Forwarder.Util; + +static class ExecutionEnvironment { - static class ExecutionEnvironment - { - public static bool SupportsStandardIO => !IsRunningAsWindowsService; + public static bool SupportsStandardIO => !IsRunningAsWindowsService; - static bool IsRunningAsWindowsService + static bool IsRunningAsWindowsService + { + get { - get - { #if WINDOWS var parent = WindowsProcess.GetParentProcess(); return parent?.ProcessName == "services"; #else - return false; + return false; #endif - } } } -} +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Util/UnclosableStreamWrapper.cs b/src/SeqCli/Forwarder/Util/UnclosableStreamWrapper.cs index 3781a124..2134c49b 100644 --- a/src/SeqCli/Forwarder/Util/UnclosableStreamWrapper.cs +++ b/src/SeqCli/Forwarder/Util/UnclosableStreamWrapper.cs @@ -15,46 +15,45 @@ using System; using System.IO; -namespace SeqCli.Forwarder.Util +namespace SeqCli.Forwarder.Util; + +class UnclosableStreamWrapper : Stream { - class UnclosableStreamWrapper : Stream + readonly Stream _stream; + + public UnclosableStreamWrapper(Stream stream) + { + _stream = stream ?? throw new ArgumentNullException(nameof(stream)); + } + + public override void Flush() + { + _stream.Flush(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + return _stream.Seek(offset, origin); + } + + public override void SetLength(long value) + { + _stream.SetLength(value); + } + + public override int Read(byte[] buffer, int offset, int count) { - readonly Stream _stream; - - public UnclosableStreamWrapper(Stream stream) - { - _stream = stream ?? throw new ArgumentNullException(nameof(stream)); - } - - public override void Flush() - { - _stream.Flush(); - } - - public override long Seek(long offset, SeekOrigin origin) - { - return _stream.Seek(offset, origin); - } - - public override void SetLength(long value) - { - _stream.SetLength(value); - } - - public override int Read(byte[] buffer, int offset, int count) - { - return _stream.Read(buffer, offset, count); - } - - public override void Write(byte[] buffer, int offset, int count) - { - _stream.Write(buffer, offset, count); - } - - public override bool CanRead => _stream.CanRead; - public override bool CanSeek => _stream.CanSeek; - public override bool CanWrite => _stream.CanWrite; - public override long Length => _stream.Length; - public override long Position { get { return _stream.Position; } set { _stream.Position = value; } } + return _stream.Read(buffer, offset, count); } + + public override void Write(byte[] buffer, int offset, int count) + { + _stream.Write(buffer, offset, count); + } + + public override bool CanRead => _stream.CanRead; + public override bool CanSeek => _stream.CanSeek; + public override bool CanWrite => _stream.CanWrite; + public override long Length => _stream.Length; + public override long Position { get { return _stream.Position; } set { _stream.Position = value; } } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs b/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs index f295aa58..9747a8d2 100644 --- a/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs +++ b/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs @@ -19,39 +19,38 @@ using SeqCli.Forwarder.Diagnostics; using Serilog.Formatting.Display; -namespace SeqCli.Forwarder.Web.Api +namespace SeqCli.Forwarder.Web.Api; + +public class ApiRootController : Controller { - public class ApiRootController : Controller - { - static readonly Encoding Encoding = new UTF8Encoding(false); - readonly MessageTemplateTextFormatter _ingestionLogFormatter; + static readonly Encoding Encoding = new UTF8Encoding(false); + readonly MessageTemplateTextFormatter _ingestionLogFormatter; - public ApiRootController(ForwarderDiagnosticConfig diagnosticConfig) - { - var template = "[{Timestamp:o} {Level:u3}] {Message}{NewLine}"; - if (diagnosticConfig.IngestionLogShowDetail) - template += "Client IP address: {ClientHostIP}{NewLine}First {StartToLog} characters of payload: {DocumentStart:l}{NewLine}{Exception}{NewLine}"; + public ApiRootController(ForwarderDiagnosticConfig diagnosticConfig) + { + var template = "[{Timestamp:o} {Level:u3}] {Message}{NewLine}"; + if (diagnosticConfig.IngestionLogShowDetail) + template += "Client IP address: {ClientHostIP}{NewLine}First {StartToLog} characters of payload: {DocumentStart:l}{NewLine}{Exception}{NewLine}"; - _ingestionLogFormatter = new MessageTemplateTextFormatter(template); - } + _ingestionLogFormatter = new MessageTemplateTextFormatter(template); + } - [HttpGet, Route("")] - public IActionResult Index() + [HttpGet, Route("")] + public IActionResult Index() + { + var events = IngestionLog.Read(); + using var log = new StringWriter(); + foreach (var logEvent in events) { - var events = IngestionLog.Read(); - using var log = new StringWriter(); - foreach (var logEvent in events) - { - _ingestionLogFormatter.Format(logEvent, log); - } - - return Content(log.ToString(), "text/plain", Encoding); + _ingestionLogFormatter.Format(logEvent, log); } - [HttpGet, Route("api")] - public IActionResult Resources() - { - return Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Encoding); - } + return Content(log.ToString(), "text/plain", Encoding); } -} + + [HttpGet, Route("api")] + public IActionResult Resources() + { + return Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Encoding); + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionController.cs b/src/SeqCli/Forwarder/Web/Api/IngestionController.cs index a9ac94bb..047a8898 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionController.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionController.cs @@ -30,217 +30,216 @@ using SeqCli.Forwarder.Schema; using SeqCli.Forwarder.Shipper; -namespace SeqCli.Forwarder.Web.Api +namespace SeqCli.Forwarder.Web.Api; + +public class IngestionController : Controller { - public class IngestionController : Controller + static readonly Encoding Encoding = new UTF8Encoding(false); + const string ClefMediaType = "application/vnd.serilog.clef"; + + readonly ActiveLogBufferMap _logBufferMap; + readonly ConnectionConfig _outputConfig; + readonly ServerResponseProxy _serverResponseProxy; + + readonly JsonSerializer _rawSerializer = JsonSerializer.Create( + new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); + + public IngestionController(ActiveLogBufferMap logBufferMap, ConnectionConfig outputConfig, ServerResponseProxy serverResponseProxy) { - static readonly Encoding Encoding = new UTF8Encoding(false); - const string ClefMediaType = "application/vnd.serilog.clef"; + _logBufferMap = logBufferMap; + _outputConfig = outputConfig; + _serverResponseProxy = serverResponseProxy; + } - readonly ActiveLogBufferMap _logBufferMap; - readonly ConnectionConfig _outputConfig; - readonly ServerResponseProxy _serverResponseProxy; + IPAddress ClientHostIP => Request.HttpContext.Connection.RemoteIpAddress!; - readonly JsonSerializer _rawSerializer = JsonSerializer.Create( - new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); + [HttpGet, Route("api/events/describe")] + public IActionResult Resources() + { + return Content("{\"Links\":{\"Raw\":\"/api/events/raw{?clef}\"}}", "application/json", Encoding); + } - public IngestionController(ActiveLogBufferMap logBufferMap, ConnectionConfig outputConfig, ServerResponseProxy serverResponseProxy) - { - _logBufferMap = logBufferMap; - _outputConfig = outputConfig; - _serverResponseProxy = serverResponseProxy; - } + [HttpPost, Route("api/events/raw")] + public async Task Ingest() + { + var clef = DefaultedBoolQuery("clef"); - IPAddress ClientHostIP => Request.HttpContext.Connection.RemoteIpAddress!; + if (clef) + return await IngestCompactFormat(); - [HttpGet, Route("api/events/describe")] - public IActionResult Resources() + var contentType = (string?) Request.Headers[HeaderNames.ContentType]; + if (contentType != null && contentType.StartsWith(ClefMediaType)) + return await IngestCompactFormat(); + + return IngestRawFormat(); + } + + IActionResult IngestRawFormat() + { + // The compact format ingestion path works with async IO. + HttpContext.Features.Get()!.AllowSynchronousIO = true; + + JObject posted; + try { - return Content("{\"Links\":{\"Raw\":\"/api/events/raw{?clef}\"}}", "application/json", Encoding); + posted = _rawSerializer.Deserialize(new JsonTextReader(new StreamReader(Request.Body))) ?? + throw new RequestProcessingException("Request body payload is JSON `null`."); } - - [HttpPost, Route("api/events/raw")] - public async Task Ingest() + catch (Exception ex) { - var clef = DefaultedBoolQuery("clef"); - - if (clef) - return await IngestCompactFormat(); - - var contentType = (string?) Request.Headers[HeaderNames.ContentType]; - if (contentType != null && contentType.StartsWith(ClefMediaType)) - return await IngestCompactFormat(); + IngestionLog.ForClient(ClientHostIP).Debug(ex,"Rejecting payload due to invalid JSON, request body could not be parsed"); + throw new RequestProcessingException("Invalid raw event JSON, body could not be parsed."); + } - return IngestRawFormat(); + if (!(posted.TryGetValue("events", StringComparison.Ordinal, out var eventsToken) || + posted.TryGetValue("Events", StringComparison.Ordinal, out eventsToken))) + { + IngestionLog.ForClient(ClientHostIP).Debug("Rejecting payload due to invalid JSON structure"); + throw new RequestProcessingException("Invalid raw event JSON, body must contain an 'Events' array."); } - IActionResult IngestRawFormat() + if (!(eventsToken is JArray events)) { - // The compact format ingestion path works with async IO. - HttpContext.Features.Get()!.AllowSynchronousIO = true; - - JObject posted; - try - { - posted = _rawSerializer.Deserialize(new JsonTextReader(new StreamReader(Request.Body))) ?? - throw new RequestProcessingException("Request body payload is JSON `null`."); - } - catch (Exception ex) - { - IngestionLog.ForClient(ClientHostIP).Debug(ex,"Rejecting payload due to invalid JSON, request body could not be parsed"); - throw new RequestProcessingException("Invalid raw event JSON, body could not be parsed."); - } + IngestionLog.ForClient(ClientHostIP).Debug("Rejecting payload due to invalid Events property structure"); + throw new RequestProcessingException("Invalid raw event JSON, the 'Events' property must be an array."); + } - if (!(posted.TryGetValue("events", StringComparison.Ordinal, out var eventsToken) || - posted.TryGetValue("Events", StringComparison.Ordinal, out eventsToken))) - { - IngestionLog.ForClient(ClientHostIP).Debug("Rejecting payload due to invalid JSON structure"); - throw new RequestProcessingException("Invalid raw event JSON, body must contain an 'Events' array."); - } + var encoded = EncodeRawEvents(events); + return Enqueue(encoded); + } - if (!(eventsToken is JArray events)) - { - IngestionLog.ForClient(ClientHostIP).Debug("Rejecting payload due to invalid Events property structure"); - throw new RequestProcessingException("Invalid raw event JSON, the 'Events' property must be an array."); - } + async Task IngestCompactFormat() + { + var rawFormat = new List(); + var reader = new StreamReader(Request.Body); - var encoded = EncodeRawEvents(events); - return Enqueue(encoded); - } + var line = await reader.ReadLineAsync(); + var lineNumber = 1; - async Task IngestCompactFormat() + while (line != null) { - var rawFormat = new List(); - var reader = new StreamReader(Request.Body); - - var line = await reader.ReadLineAsync(); - var lineNumber = 1; - - while (line != null) + if (!string.IsNullOrWhiteSpace(line)) { - if (!string.IsNullOrWhiteSpace(line)) + JObject item; + try { - JObject item; - try - { - item = _rawSerializer.Deserialize(new JsonTextReader(new StringReader(line))) ?? - throw new RequestProcessingException("Request body payload is JSON `null`."); - } - catch (Exception ex) - { - IngestionLog.ForPayload(ClientHostIP, line).Debug(ex, "Rejecting CLEF payload due to invalid JSON, item could not be parsed"); - throw new RequestProcessingException($"Invalid raw event JSON, item on line {lineNumber} could not be parsed."); - } - - if (!EventSchema.FromClefFormat(lineNumber, item, out var evt, out var err)) - { - IngestionLog.ForPayload(ClientHostIP, line).Debug("Rejecting CLEF payload due to invalid event JSON structure: {NormalizationError}", err); - throw new RequestProcessingException(err); - } + item = _rawSerializer.Deserialize(new JsonTextReader(new StringReader(line))) ?? + throw new RequestProcessingException("Request body payload is JSON `null`."); + } + catch (Exception ex) + { + IngestionLog.ForPayload(ClientHostIP, line).Debug(ex, "Rejecting CLEF payload due to invalid JSON, item could not be parsed"); + throw new RequestProcessingException($"Invalid raw event JSON, item on line {lineNumber} could not be parsed."); + } - rawFormat.Add(evt); + if (!EventSchema.FromClefFormat(lineNumber, item, out var evt, out var err)) + { + IngestionLog.ForPayload(ClientHostIP, line).Debug("Rejecting CLEF payload due to invalid event JSON structure: {NormalizationError}", err); + throw new RequestProcessingException(err); } - line = await reader.ReadLineAsync(); - ++lineNumber; + rawFormat.Add(evt); } - var encoded = EncodeRawEvents(rawFormat); - return Enqueue(encoded); + line = await reader.ReadLineAsync(); + ++lineNumber; } - byte[][] EncodeRawEvents(ICollection events) + var encoded = EncodeRawEvents(rawFormat); + return Enqueue(encoded); + } + + byte[][] EncodeRawEvents(ICollection events) + { + var encoded = new byte[events.Count][]; + var i = 0; + foreach (var e in events) { - var encoded = new byte[events.Count][]; - var i = 0; - foreach (var e in events) + var s = e.ToString(Formatting.None); + var payload = Encoding.UTF8.GetBytes(s); + + if (payload.Length > (int) _outputConfig.EventBodyLimitBytes) { - var s = e.ToString(Formatting.None); - var payload = Encoding.UTF8.GetBytes(s); + IngestionLog.ForPayload(ClientHostIP, s).Debug("An oversized event was dropped"); - if (payload.Length > (int) _outputConfig.EventBodyLimitBytes) + var jo = e as JObject; + // ReSharper disable SuspiciousTypeConversion.Global + var timestamp = (string?) (dynamic?) jo?.GetValue("Timestamp") ?? DateTime.UtcNow.ToString("o"); + var level = (string?) (dynamic?) jo?.GetValue("Level") ?? "Warning"; + + if (jo != null) { - IngestionLog.ForPayload(ClientHostIP, s).Debug("An oversized event was dropped"); + jo.Remove("Timestamp"); + jo.Remove("Level"); + } - var jo = e as JObject; - // ReSharper disable SuspiciousTypeConversion.Global - var timestamp = (string?) (dynamic?) jo?.GetValue("Timestamp") ?? DateTime.UtcNow.ToString("o"); - var level = (string?) (dynamic?) jo?.GetValue("Level") ?? "Warning"; + var startToLog = (int) Math.Min(_outputConfig.EventBodyLimitBytes / 2, 1024); + var compactPrefix = e.ToString(Formatting.None).Substring(0, startToLog); - if (jo != null) + encoded[i] = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new + { + Timestamp = timestamp, + MessageTemplate = "Seq Forwarder received and dropped an oversized event", + Level = level, + Properties = new { - jo.Remove("Timestamp"); - jo.Remove("Level"); + Partial = compactPrefix, + Environment.MachineName, + _outputConfig.EventBodyLimitBytes, + PayloadBytes = payload.Length } - - var startToLog = (int) Math.Min(_outputConfig.EventBodyLimitBytes / 2, 1024); - var compactPrefix = e.ToString(Formatting.None).Substring(0, startToLog); - - encoded[i] = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new - { - Timestamp = timestamp, - MessageTemplate = "Seq Forwarder received and dropped an oversized event", - Level = level, - Properties = new - { - Partial = compactPrefix, - Environment.MachineName, - _outputConfig.EventBodyLimitBytes, - PayloadBytes = payload.Length - } - })); - } - else - { - encoded[i] = payload; - } - - i++; + })); + } + else + { + encoded[i] = payload; } - return encoded; + i++; } + + return encoded; + } - IActionResult Enqueue(byte[][] encodedEvents) - { - var apiKey = GetRequestApiKeyToken(); - _logBufferMap.GetLogBuffer(apiKey).Enqueue(encodedEvents); + IActionResult Enqueue(byte[][] encodedEvents) + { + var apiKey = GetRequestApiKeyToken(); + _logBufferMap.GetLogBuffer(apiKey).Enqueue(encodedEvents); - var response = Content(_serverResponseProxy.GetResponseText(apiKey), "application/json", Encoding); - response.StatusCode = (int)HttpStatusCode.Created; - return response; - } + var response = Content(_serverResponseProxy.GetResponseText(apiKey), "application/json", Encoding); + response.StatusCode = (int)HttpStatusCode.Created; + return response; + } - string? GetRequestApiKeyToken() - { - var apiKeyToken = Request.Headers[SeqApi.ApiKeyHeaderName].FirstOrDefault(); + string? GetRequestApiKeyToken() + { + var apiKeyToken = Request.Headers[SeqApi.ApiKeyHeaderName].FirstOrDefault(); - if (string.IsNullOrWhiteSpace(apiKeyToken)) - apiKeyToken = Request.Query["apiKey"]; + if (string.IsNullOrWhiteSpace(apiKeyToken)) + apiKeyToken = Request.Query["apiKey"]; - var normalized = apiKeyToken?.Trim(); - if (string.IsNullOrEmpty(normalized)) - return null; + var normalized = apiKeyToken?.Trim(); + if (string.IsNullOrEmpty(normalized)) + return null; - return normalized; - } + return normalized; + } - bool DefaultedBoolQuery(string queryParameterName) - { - var parameter = Request.Query[queryParameterName]; - if (parameter.Count != 1) - return false; + bool DefaultedBoolQuery(string queryParameterName) + { + var parameter = Request.Query[queryParameterName]; + if (parameter.Count != 1) + return false; - var value = (string?) parameter; + var value = (string?) parameter; - if (value == "" && ( + if (value == "" && ( Request.QueryString.Value!.Contains($"&{queryParameterName}=") || Request.QueryString.Value.Contains($"?{queryParameterName}="))) - { - return false; - } - - return "true".Equals(value, StringComparison.OrdinalIgnoreCase) || value == "" || value == queryParameterName; + { + return false; } + + return "true".Equals(value, StringComparison.OrdinalIgnoreCase) || value == "" || value == queryParameterName; } -} +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Host/ServerService.cs b/src/SeqCli/Forwarder/Web/Host/ServerService.cs index 509d86e6..9118eb24 100644 --- a/src/SeqCli/Forwarder/Web/Host/ServerService.cs +++ b/src/SeqCli/Forwarder/Web/Host/ServerService.cs @@ -18,50 +18,49 @@ using SeqCli.Forwarder.Multiplexing; using Serilog; -namespace SeqCli.Forwarder.Web.Host +namespace SeqCli.Forwarder.Web.Host; + +class ServerService { - class ServerService - { - readonly ActiveLogBufferMap _logBufferMap; - readonly IHost _host; - readonly string _listenUri; + readonly ActiveLogBufferMap _logBufferMap; + readonly IHost _host; + readonly string _listenUri; - public ServerService(ActiveLogBufferMap logBufferMap, IHost host, string listenUri) - { - _logBufferMap = logBufferMap; - _host = host; - _listenUri = listenUri; - } + public ServerService(ActiveLogBufferMap logBufferMap, IHost host, string listenUri) + { + _logBufferMap = logBufferMap; + _host = host; + _listenUri = listenUri; + } - public void Start() + public void Start() + { + try { - try - { - Log.Debug("Starting HTTP server..."); + Log.Debug("Starting HTTP server..."); - _host.Start(); + _host.Start(); - Log.Information("Seq Forwarder listening on {ListenUri}", _listenUri); - IngestionLog.Log.Debug("Seq Forwarder is accepting events"); + Log.Information("Seq Forwarder listening on {ListenUri}", _listenUri); + IngestionLog.Log.Debug("Seq Forwarder is accepting events"); - _logBufferMap.Load(); - _logBufferMap.Start(); - } - catch (Exception ex) - { - Log.Fatal(ex, "Error running the server application"); - throw; - } + _logBufferMap.Load(); + _logBufferMap.Start(); } - - public void Stop() + catch (Exception ex) { - Log.Debug("Seq Forwarder stopping"); + Log.Fatal(ex, "Error running the server application"); + throw; + } + } - _host.StopAsync().Wait(); - _logBufferMap.Stop(); + public void Stop() + { + Log.Debug("Seq Forwarder stopping"); - Log.Information("Seq Forwarder stopped cleanly"); - } + _host.StopAsync().Wait(); + _logBufferMap.Stop(); + + Log.Information("Seq Forwarder stopped cleanly"); } -} +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Host/Startup.cs b/src/SeqCli/Forwarder/Web/Host/Startup.cs index 56d6fe9f..804e2065 100644 --- a/src/SeqCli/Forwarder/Web/Host/Startup.cs +++ b/src/SeqCli/Forwarder/Web/Host/Startup.cs @@ -2,39 +2,38 @@ using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.DependencyInjection; -namespace SeqCli.Forwarder.Web.Host +namespace SeqCli.Forwarder.Web.Host; + +class Startup { - class Startup + public void ConfigureServices(IServiceCollection serviceCollection) { - public void ConfigureServices(IServiceCollection serviceCollection) - { - serviceCollection.AddMvc(); - } + serviceCollection.AddMvc(); + } - public void Configure(IApplicationBuilder app) + public void Configure(IApplicationBuilder app) + { + app.Use(async (context, next) => { - app.Use(async (context, next) => + try { - try - { - await next(); - } - catch (RequestProcessingException rex) - { - if (context.Response.HasStarted) - throw; - - context.Response.StatusCode = (int)rex.StatusCode; - context.Response.ContentType = "text/plain; charset=UTF-8"; - await context.Response.Body.WriteAsync(Encoding.UTF8.GetBytes(rex.Message)); - await context.Response.CompleteAsync(); - } - }); - app.UseRouting(); - app.UseEndpoints(endpoints => + await next(); + } + catch (RequestProcessingException rex) { - endpoints.MapControllers(); - }); - } + if (context.Response.HasStarted) + throw; + + context.Response.StatusCode = (int)rex.StatusCode; + context.Response.ContentType = "text/plain; charset=UTF-8"; + await context.Response.Body.WriteAsync(Encoding.UTF8.GetBytes(rex.Message)); + await context.Response.CompleteAsync(); + } + }); + app.UseRouting(); + app.UseEndpoints(endpoints => + { + endpoints.MapControllers(); + }); } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/RequestProcessingException.cs b/src/SeqCli/Forwarder/Web/RequestProcessingException.cs index faa95e43..bfef07af 100644 --- a/src/SeqCli/Forwarder/Web/RequestProcessingException.cs +++ b/src/SeqCli/Forwarder/Web/RequestProcessingException.cs @@ -15,16 +15,15 @@ using System; using System.Net; -namespace SeqCli.Forwarder.Web +namespace SeqCli.Forwarder.Web; + +class RequestProcessingException : Exception { - class RequestProcessingException : Exception + public RequestProcessingException(string message, HttpStatusCode statusCode = HttpStatusCode.BadRequest) + : base(message) { - public RequestProcessingException(string message, HttpStatusCode statusCode = HttpStatusCode.BadRequest) - : base(message) - { - StatusCode = statusCode; - } - - public HttpStatusCode StatusCode { get; } + StatusCode = statusCode; } -} + + public HttpStatusCode StatusCode { get; } +} \ No newline at end of file diff --git a/src/SeqCli/Syntax/QueryBuilder.cs b/src/SeqCli/Syntax/QueryBuilder.cs index 30b6aeaf..b2338ec4 100644 --- a/src/SeqCli/Syntax/QueryBuilder.cs +++ b/src/SeqCli/Syntax/QueryBuilder.cs @@ -21,10 +21,10 @@ namespace SeqCli.Syntax; class QueryBuilder { - readonly List<(string, string)> _columns = new List<(string, string)>(); - readonly List _where = new List(); - readonly List _groupBy = new List(); - readonly List _having = new List(); + readonly List<(string, string)> _columns = new(); + readonly List _where = new(); + readonly List _groupBy = new(); + readonly List _having = new(); public void Select(string value, string label) { diff --git a/test/SeqCli.EndToEnd/Args.cs b/test/SeqCli.EndToEnd/Args.cs index 258e17f8..f3b0bd97 100644 --- a/test/SeqCli.EndToEnd/Args.cs +++ b/test/SeqCli.EndToEnd/Args.cs @@ -18,7 +18,7 @@ public Regex[] TestCases() => _args .ToArray(); // Simple replacement so `Events.*` becomes `Events\..*` - static Regex ToArgRegex(string arg) => new Regex(arg.Replace(".", "\\.").Replace("*", ".*")); + static Regex ToArgRegex(string arg) => new(arg.Replace(".", "\\.").Replace("*", ".*")); public bool Multiuser() => _args.Any(a => a == "--license-certificate-stdin"); diff --git a/test/SeqCli.EndToEnd/Support/CaptiveProcess.cs b/test/SeqCli.EndToEnd/Support/CaptiveProcess.cs index a08bd598..1f7bb834 100644 --- a/test/SeqCli.EndToEnd/Support/CaptiveProcess.cs +++ b/test/SeqCli.EndToEnd/Support/CaptiveProcess.cs @@ -12,11 +12,11 @@ public sealed class CaptiveProcess : ITestProcess, IDisposable readonly string _stopCommandFullExePath; readonly string _stopCommandArgs; readonly Process _process; - readonly ManualResetEvent _outputComplete = new ManualResetEvent(false); - readonly ManualResetEvent _errorComplete = new ManualResetEvent(false); + readonly ManualResetEvent _outputComplete = new(false); + readonly ManualResetEvent _errorComplete = new(false); - readonly object _sync = new object(); - readonly StringWriter _output = new StringWriter(); + readonly object _sync = new(); + readonly StringWriter _output = new(); public CaptiveProcess( string fullExePath, diff --git a/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs index 8231d82b..11db09d1 100644 --- a/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs +++ b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs @@ -6,77 +6,76 @@ using SeqCli.Tests.Support; using Xunit; -namespace SeqCli.Tests.Forwarder.Multiplexing -{ - public class ActiveLogBufferMapTests - { - [Fact] - public void AnEmptyMapCreatesNoFiles() - { - using var tmp = new TempFolder("Buffer"); - using var map = CreateActiveLogBufferMap(tmp); - Assert.Empty(Directory.GetFileSystemEntries(tmp.Path)); - } +namespace SeqCli.Tests.Forwarder.Multiplexing; - [Fact] - public void TheDefaultBufferWritesDataInTheBufferRoot() - { - using var tmp = new TempFolder("Buffer"); - using var map = CreateActiveLogBufferMap(tmp); - var entry = map.GetLogBuffer(null); - Assert.NotNull(entry); - Assert.True(File.Exists(Path.Combine(tmp.Path, "data.mdb"))); - Assert.Empty(Directory.GetDirectories(tmp.Path)); - Assert.Same(entry, map.GetLogBuffer(null)); - } +public class ActiveLogBufferMapTests +{ + [Fact] + public void AnEmptyMapCreatesNoFiles() + { + using var tmp = new TempFolder("Buffer"); + using var map = CreateActiveLogBufferMap(tmp); + Assert.Empty(Directory.GetFileSystemEntries(tmp.Path)); + } - [Fact] - public void ApiKeySpecificBuffersWriteDataToSubfolders() - { - using var tmp = new TempFolder("Buffer"); - using var map = CreateActiveLogBufferMap(tmp); - string key1 = Some.ApiKey(), key2 = Some.ApiKey(); - var entry1 = map.GetLogBuffer(key1); - var entry2 = map.GetLogBuffer(key2); + [Fact] + public void TheDefaultBufferWritesDataInTheBufferRoot() + { + using var tmp = new TempFolder("Buffer"); + using var map = CreateActiveLogBufferMap(tmp); + var entry = map.GetLogBuffer(null); + Assert.NotNull(entry); + Assert.True(File.Exists(Path.Combine(tmp.Path, "data.mdb"))); + Assert.Empty(Directory.GetDirectories(tmp.Path)); + Assert.Same(entry, map.GetLogBuffer(null)); + } - Assert.NotNull(entry1); - Assert.NotNull(entry2); - Assert.Same(entry1, map.GetLogBuffer(key1)); - Assert.NotSame(entry1, entry2); - var subdirs = Directory.GetDirectories(tmp.Path); - Assert.Equal(2, subdirs.Length); - Assert.True(File.Exists(Path.Combine(subdirs[0], "data.mdb"))); - Assert.True(File.Exists(Path.Combine(subdirs[0], ".apikey"))); - } + [Fact] + public void ApiKeySpecificBuffersWriteDataToSubfolders() + { + using var tmp = new TempFolder("Buffer"); + using var map = CreateActiveLogBufferMap(tmp); + string key1 = Some.ApiKey(), key2 = Some.ApiKey(); + var entry1 = map.GetLogBuffer(key1); + var entry2 = map.GetLogBuffer(key2); - [Fact] - public void EntriesSurviveReloads() - { - var apiKey = Some.ApiKey(); - var value = Some.Bytes(100); + Assert.NotNull(entry1); + Assert.NotNull(entry2); + Assert.Same(entry1, map.GetLogBuffer(key1)); + Assert.NotSame(entry1, entry2); + var subdirs = Directory.GetDirectories(tmp.Path); + Assert.Equal(2, subdirs.Length); + Assert.True(File.Exists(Path.Combine(subdirs[0], "data.mdb"))); + Assert.True(File.Exists(Path.Combine(subdirs[0], ".apikey"))); + } - using var tmp = new TempFolder("Buffer"); - using (var map = CreateActiveLogBufferMap(tmp)) - { - map.GetLogBuffer(null).Enqueue([value]); - map.GetLogBuffer(apiKey).Enqueue([value]); - } + [Fact] + public void EntriesSurviveReloads() + { + var apiKey = Some.ApiKey(); + var value = Some.Bytes(100); - using (var map = CreateActiveLogBufferMap(tmp)) - { - var first = map.GetLogBuffer(null).Peek(0).Single(); - var second = map.GetLogBuffer(apiKey).Peek(0).Single(); - Assert.Equal(value, first.Value); - Assert.Equal(value, second.Value); - } + using var tmp = new TempFolder("Buffer"); + using (var map = CreateActiveLogBufferMap(tmp)) + { + map.GetLogBuffer(null).Enqueue([value]); + map.GetLogBuffer(apiKey).Enqueue([value]); } - static ActiveLogBufferMap CreateActiveLogBufferMap(TempFolder tmp) + using (var map = CreateActiveLogBufferMap(tmp)) { - var config = new SeqCliConfig(); - var map = new ActiveLogBufferMap(tmp.Path, config.Forwarder.Storage, config.Connection, new InertLogShipperFactory(), StringDataProtector.CreatePlatformDefault()); - map.Load(); - return map; + var first = map.GetLogBuffer(null).Peek(0).Single(); + var second = map.GetLogBuffer(apiKey).Peek(0).Single(); + Assert.Equal(value, first.Value); + Assert.Equal(value, second.Value); } } -} + + static ActiveLogBufferMap CreateActiveLogBufferMap(TempFolder tmp) + { + var config = new SeqCliConfig(); + var map = new ActiveLogBufferMap(tmp.Path, config.Forwarder.Storage, config.Connection, new InertLogShipperFactory(), StringDataProtector.CreatePlatformDefault()); + map.Load(); + return map; + } +} \ No newline at end of file diff --git a/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs b/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs index 05f1a7f3..8d1abcba 100644 --- a/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs +++ b/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs @@ -4,70 +4,69 @@ using SeqCli.Forwarder.Schema; using Xunit; -namespace SeqCli.Tests.Forwarder.Schema +namespace SeqCli.Tests.Forwarder.Schema; + +public class EventSchemaTests { - public class EventSchemaTests - { - static readonly JsonSerializer RawSerializer = JsonSerializer.Create( - new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); + static readonly JsonSerializer RawSerializer = JsonSerializer.Create( + new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); - [Fact] - public void ClefNormalizationAcceptsDuplicateRenderings() - { - var payload = "{\"@t\": \"2015-05-09T12:09:08.12345Z\"," + - " \"@mt\": \"{A:000} and {A:000}\"," + - " \"@r\": [\"424\",\"424\"]}"; + [Fact] + public void ClefNormalizationAcceptsDuplicateRenderings() + { + var payload = "{\"@t\": \"2015-05-09T12:09:08.12345Z\"," + + " \"@mt\": \"{A:000} and {A:000}\"," + + " \"@r\": [\"424\",\"424\"]}"; - AssertCanNormalizeClef(payload); - } + AssertCanNormalizeClef(payload); + } - [Fact] - public void ClefNormalizationPropagatesRenderings() - { - const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12,\"@r\":[\"012\"]}"; - var evt = AssertCanNormalizeClef(payload); - Assert.Single(evt.Renderings); - } + [Fact] + public void ClefNormalizationPropagatesRenderings() + { + const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12,\"@r\":[\"012\"]}"; + var evt = AssertCanNormalizeClef(payload); + Assert.Single(evt.Renderings); + } - [Fact] - public void ClefNormalizationIgnoresMissingRenderings() - { - const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12}"; - AssertCanNormalizeClef(payload); - } + [Fact] + public void ClefNormalizationIgnoresMissingRenderings() + { + const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12}"; + AssertCanNormalizeClef(payload); + } - [Fact] - public void ClefNormalizationFixesTooFewRenderings1() - { - const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12,\"@r\":[]}"; - var evt = AssertCanNormalizeClef(payload); - Assert.Null(evt.Renderings); - } + [Fact] + public void ClefNormalizationFixesTooFewRenderings1() + { + const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12,\"@r\":[]}"; + var evt = AssertCanNormalizeClef(payload); + Assert.Null(evt.Renderings); + } - [Fact] - public void ClefNormalizationFixesTooFewRenderings2() - { - const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000} {Q:x}!\",\"P\":12,\"@r\":[\"012\"]}"; - var evt = AssertCanNormalizeClef(payload); - Assert.Null(evt.Renderings); - } + [Fact] + public void ClefNormalizationFixesTooFewRenderings2() + { + const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000} {Q:x}!\",\"P\":12,\"@r\":[\"012\"]}"; + var evt = AssertCanNormalizeClef(payload); + Assert.Null(evt.Renderings); + } - [Fact] - public void ClefNormalizationIgnoresTooManyRenderings() - { - const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12,\"@r\":[\"012\",\"013\"]}"; - var evt = AssertCanNormalizeClef(payload); - Assert.Null(evt.Renderings); - } + [Fact] + public void ClefNormalizationIgnoresTooManyRenderings() + { + const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12,\"@r\":[\"012\",\"013\"]}"; + var evt = AssertCanNormalizeClef(payload); + Assert.Null(evt.Renderings); + } - static dynamic AssertCanNormalizeClef(string payload) - { - var jo = RawSerializer.Deserialize(new JsonTextReader(new StringReader(payload)))!; + static dynamic AssertCanNormalizeClef(string payload) + { + var jo = RawSerializer.Deserialize(new JsonTextReader(new StringReader(payload)))!; - var valid = EventSchema.FromClefFormat(1, jo, out var rawFormat, out var error); - Assert.True(valid, error); - Assert.NotNull(rawFormat); - return rawFormat!; - } + var valid = EventSchema.FromClefFormat(1, jo, out var rawFormat, out var error); + Assert.True(valid, error); + Assert.NotNull(rawFormat); + return rawFormat!; } } \ No newline at end of file diff --git a/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs b/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs index 217216a2..107a6973 100644 --- a/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs +++ b/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs @@ -2,46 +2,45 @@ using SeqCli.Tests.Support; using Xunit; -namespace SeqCli.Tests.Forwarder.Shipper +namespace SeqCli.Tests.Forwarder.Shipper; + +public class ServerResponseProxyTests { - public class ServerResponseProxyTests + [Fact] + public void WhenNoResponseRecordedEmptyIsReturned() { - [Fact] - public void WhenNoResponseRecordedEmptyIsReturned() - { - var proxy = new ServerResponseProxy(); - var response = proxy.GetResponseText(Some.ApiKey()); - Assert.Equal("{}", response); - } + var proxy = new ServerResponseProxy(); + var response = proxy.GetResponseText(Some.ApiKey()); + Assert.Equal("{}", response); + } - [Fact] - public void WhenApiKeysDontMatchEmptyResponseReturned() - { - var proxy = new ServerResponseProxy(); - proxy.SuccessResponseReturned(Some.ApiKey(), "this is never used"); - var response = proxy.GetResponseText(Some.ApiKey()); - Assert.Equal("{}", response); - } + [Fact] + public void WhenApiKeysDontMatchEmptyResponseReturned() + { + var proxy = new ServerResponseProxy(); + proxy.SuccessResponseReturned(Some.ApiKey(), "this is never used"); + var response = proxy.GetResponseText(Some.ApiKey()); + Assert.Equal("{}", response); + } - [Fact] - public void WhenApiKeysMatchTheResponseIsReturned() - { - var proxy = new ServerResponseProxy(); - var apiKey = Some.ApiKey(); - var responseText = "some response"; - proxy.SuccessResponseReturned(apiKey, responseText); - var response = proxy.GetResponseText(apiKey); - Assert.Equal(responseText, response); - } + [Fact] + public void WhenApiKeysMatchTheResponseIsReturned() + { + var proxy = new ServerResponseProxy(); + var apiKey = Some.ApiKey(); + var responseText = "some response"; + proxy.SuccessResponseReturned(apiKey, responseText); + var response = proxy.GetResponseText(apiKey); + Assert.Equal(responseText, response); + } - [Fact] - public void NullApiKeysAreConsideredMatching() - { - var proxy = new ServerResponseProxy(); - var responseText = "some response"; - proxy.SuccessResponseReturned(null, responseText); - var response = proxy.GetResponseText(null); - Assert.Equal(responseText, response); - } + [Fact] + public void NullApiKeysAreConsideredMatching() + { + var proxy = new ServerResponseProxy(); + var responseText = "some response"; + proxy.SuccessResponseReturned(null, responseText); + var response = proxy.GetResponseText(null); + Assert.Equal(responseText, response); } -} +} \ No newline at end of file diff --git a/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs b/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs index 9ad315ac..cee028fc 100644 --- a/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs +++ b/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs @@ -3,148 +3,147 @@ using SeqCli.Tests.Support; using Xunit; -namespace SeqCli.Tests.Forwarder.Storage +namespace SeqCli.Tests.Forwarder.Storage; + +public class LogBufferTests { - public class LogBufferTests + const ulong DefaultBufferSize = 10 * 1024 * 1024; + + [Fact] + public void ANewLogBufferIsEmpty() { - const ulong DefaultBufferSize = 10 * 1024 * 1024; + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + var contents = buffer.Peek((int)DefaultBufferSize); + Assert.Empty(contents); + } - [Fact] - public void ANewLogBufferIsEmpty() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - var contents = buffer.Peek((int)DefaultBufferSize); - Assert.Empty(contents); - } - - [Fact] - public void PeekingDoesNotChangeState() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - buffer.Enqueue([Some.Bytes(140)]); + [Fact] + public void PeekingDoesNotChangeState() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + buffer.Enqueue([Some.Bytes(140)]); - var contents = buffer.Peek((int)DefaultBufferSize); - Assert.Single(contents); + var contents = buffer.Peek((int)DefaultBufferSize); + Assert.Single(contents); - var remainder = buffer.Peek((int)DefaultBufferSize); - Assert.Single(remainder); - } + var remainder = buffer.Peek((int)DefaultBufferSize); + Assert.Single(remainder); + } - [Fact] - public void EnqueuedEntriesAreDequeuedFifo() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2]); - buffer.Enqueue([a3]); - - var contents = buffer.Peek((int)DefaultBufferSize); - - Assert.Equal(3, contents.Length); - Assert.Equal(a1, contents[0].Value); - Assert.Equal(a2, contents[1].Value); - Assert.Equal(a3, contents[2].Value); - } - - [Fact] - public void EntriesOverLimitArePurgedFifo() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), 4096); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); + [Fact] + public void EnqueuedEntriesAreDequeuedFifo() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue([a1, a2]); + buffer.Enqueue([a3]); + + var contents = buffer.Peek((int)DefaultBufferSize); + + Assert.Equal(3, contents.Length); + Assert.Equal(a1, contents[0].Value); + Assert.Equal(a2, contents[1].Value); + Assert.Equal(a3, contents[2].Value); + } - var contents = buffer.Peek((int)DefaultBufferSize); + [Fact] + public void EntriesOverLimitArePurgedFifo() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), 4096); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue([a1, a2, a3]); - Assert.Equal(2, contents.Length); - Assert.Equal(a2, contents[0].Value); - Assert.Equal(a3, contents[1].Value); - } + var contents = buffer.Peek((int)DefaultBufferSize); - [Fact] - public void SizeHintLimitsDequeuedEventCount() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); + Assert.Equal(2, contents.Length); + Assert.Equal(a2, contents[0].Value); + Assert.Equal(a3, contents[1].Value); + } - var contents = buffer.Peek(300); + [Fact] + public void SizeHintLimitsDequeuedEventCount() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue([a1, a2, a3]); - Assert.Equal(2, contents.Length); - Assert.Equal(a1, contents[0].Value); - Assert.Equal(a2, contents[1].Value); - } + var contents = buffer.Peek(300); - [Fact] - public void AtLeastOneEventIsAlwaysDequeued() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); + Assert.Equal(2, contents.Length); + Assert.Equal(a1, contents[0].Value); + Assert.Equal(a2, contents[1].Value); + } - var contents = buffer.Peek(30); + [Fact] + public void AtLeastOneEventIsAlwaysDequeued() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue([a1, a2, a3]); - Assert.Single(contents); - Assert.Equal(a1, contents[0].Value); - } + var contents = buffer.Peek(30); - [Fact] - public void GivingTheLastSeenEventKeyRemovesPrecedingEvents() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); + Assert.Single(contents); + Assert.Equal(a1, contents[0].Value); + } - var contents = buffer.Peek(420); - Assert.Equal(3, contents.Length); + [Fact] + public void GivingTheLastSeenEventKeyRemovesPrecedingEvents() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue([a1, a2, a3]); - buffer.Dequeue(contents[2].Key); + var contents = buffer.Peek(420); + Assert.Equal(3, contents.Length); - var remaining = buffer.Peek(420); - Assert.Empty(remaining); - } + buffer.Dequeue(contents[2].Key); - [Fact] - public void GivingTheLastSeeEventKeyLeavesSuccessiveEvents() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); + var remaining = buffer.Peek(420); + Assert.Empty(remaining); + } + + [Fact] + public void GivingTheLastSeeEventKeyLeavesSuccessiveEvents() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue([a1, a2, a3]); - var contents = buffer.Peek(30); - Assert.Single(contents); + var contents = buffer.Peek(30); + Assert.Single(contents); - buffer.Enqueue([Some.Bytes(140)]); + buffer.Enqueue([Some.Bytes(140)]); - buffer.Dequeue(contents[0].Key); + buffer.Dequeue(contents[0].Key); - var remaining = buffer.Peek(420); - Assert.Equal(3, remaining.Length); - } + var remaining = buffer.Peek(420); + Assert.Equal(3, remaining.Length); + } - [Fact] - public void EnumerationIsInOrder() + [Fact] + public void EnumerationIsInOrder() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue([a1, a2, a3]); + + var contents = new List(); + buffer.Enumerate((k, v) => { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); - - var contents = new List(); - buffer.Enumerate((k, v) => - { - contents.Add(v); - }); - - Assert.Equal(3, contents.Count); - Assert.Equal(new[] { a1, a2, a3 }, contents); - } + contents.Add(v); + }); + + Assert.Equal(3, contents.Count); + Assert.Equal(new[] { a1, a2, a3 }, contents); } -} +} \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs b/test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs index 8deac0f2..cfcb203e 100644 --- a/test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs +++ b/test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs @@ -35,7 +35,7 @@ public void TheTrailingIndentPatternDoesNotMatchLinesStartingWithWhitespace() Assert.Equal(frame, remainder); } - static NameValueExtractor ClassMethodPattern { get; } = new NameValueExtractor(new[] + static NameValueExtractor ClassMethodPattern { get; } = new(new[] { new SimplePatternElement(Matchers.Identifier, "class"), new SimplePatternElement(Matchers.LiteralText(".")), diff --git a/test/SeqCli.Tests/Support/TempFolder.cs b/test/SeqCli.Tests/Support/TempFolder.cs index 498ed571..968fd857 100644 --- a/test/SeqCli.Tests/Support/TempFolder.cs +++ b/test/SeqCli.Tests/Support/TempFolder.cs @@ -5,47 +5,46 @@ using System.IO; using System.Runtime.CompilerServices; -namespace SeqCli.Tests.Support +namespace SeqCli.Tests.Support; + +class TempFolder : IDisposable { - class TempFolder : IDisposable - { - static readonly Guid Session = Guid.NewGuid(); + static readonly Guid Session = Guid.NewGuid(); - public TempFolder(string name) - { - Path = System.IO.Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), - "Seq.Forwarder.Tests", - Session.ToString("n"), - name); + public TempFolder(string name) + { + Path = System.IO.Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), + "Seq.Forwarder.Tests", + Session.ToString("n"), + name); - Directory.CreateDirectory(Path); - } + Directory.CreateDirectory(Path); + } - public string Path { get; } + public string Path { get; } - public void Dispose() + public void Dispose() + { + try { - try - { - if (Directory.Exists(Path)) - Directory.Delete(Path, true); - } - catch (Exception ex) - { - Debug.WriteLine(ex); - } + if (Directory.Exists(Path)) + Directory.Delete(Path, true); } - - public static TempFolder ForCaller([CallerMemberName] string? caller = null) + catch (Exception ex) { - if (caller == null) throw new ArgumentNullException(nameof(caller)); - return new TempFolder(caller); + Debug.WriteLine(ex); } + } - public string AllocateFilename(string? ext = null) - { - return System.IO.Path.Combine(Path, Guid.NewGuid().ToString("n") + "." + (ext ?? "tmp")); - } + public static TempFolder ForCaller([CallerMemberName] string? caller = null) + { + if (caller == null) throw new ArgumentNullException(nameof(caller)); + return new TempFolder(caller); + } + + public string AllocateFilename(string? ext = null) + { + return System.IO.Path.Combine(Path, Guid.NewGuid().ToString("n") + "." + (ext ?? "tmp")); } -} +} \ No newline at end of file From 892a7e5742fb1eff863e5ffa79e46b7442bebf5f Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 29 Feb 2024 15:23:06 +1000 Subject: [PATCH 16/51] Improve some command wording --- src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs | 2 +- src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs | 2 +- src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs | 2 +- src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs | 2 +- src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs | 2 +- src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs | 2 +- src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs | 7 ++++++- src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs | 2 +- 8 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs index 4fc70ed0..3eeda939 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs @@ -33,7 +33,7 @@ namespace Seq.Forwarder.Cli.Commands { - [Command("forwarder", "install", "Install the Seq Forwarder as a Windows service")] + [Command("forwarder", "install", "Install the forwarder as a Windows service")] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class InstallCommand : Command { diff --git a/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs index bf04e666..c53f9b70 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs @@ -26,7 +26,7 @@ namespace Seq.Forwarder.Cli.Commands { - [Command("forwarder", "restart", "Restart the Windows service")] + [Command("forwarder", "restart", "Restart the forwarder Windows service")] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class RestartCommand : Command { diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index e743f174..3c94225e 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -36,7 +36,7 @@ namespace SeqCli.Cli.Commands.Forwarder; -[Command("forwarder", "run", "Run the server interactively")] +[Command("forwarder", "run", "Listen on an HTTP endpoint and forward ingested logs to Seq")] class RunCommand : Command { readonly StoragePathFeature _storagePath; diff --git a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs index 150c9f6d..5353e142 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs @@ -24,7 +24,7 @@ namespace Seq.Forwarder.Cli.Commands { - [Command("forwarder", "start", "Start the Windows service")] + [Command("forwarder", "start", "Start the forwarder Windows service")] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class StartCommand : Command { diff --git a/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs index 40b05669..1b0aa06d 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs @@ -24,7 +24,7 @@ namespace Seq.Forwarder.Cli.Commands { - [Command("forwarder", "status", "Show the status of the Seq Forwarder service")] + [Command("forwarder", "status", "Show the status of the forwarder Windows service")] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class StatusCommand : Command { diff --git a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs index d5ee2f1a..a003bb22 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs @@ -24,7 +24,7 @@ namespace Seq.Forwarder.Cli.Commands { - [Command("forwarder", "stop", "Stop the Windows service")] + [Command("forwarder", "stop", "Stop the forwarder Windows service")] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class StopCommand : Command { diff --git a/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs index df5130da..c73b37d6 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs @@ -20,20 +20,25 @@ namespace SeqCli.Cli.Commands.Forwarder; -[Command("forwarder", "truncate", "Clear the log buffer contents")] +[Command("forwarder", "truncate", "Empty the forwarder's persistent log buffer")] class TruncateCommand : Command { readonly StoragePathFeature _storagePath; + readonly ConfirmFeature _confirm; public TruncateCommand() { _storagePath = Enable(); + _confirm = Enable(); } protected override async Task Run(string[] args) { try { + if (!_confirm.TryConfirm("All data in the forwarder's log buffer will be deleted. This cannot be undone.")) + return 1; + ActiveLogBufferMap.Truncate(_storagePath.BufferPath); return 0; } diff --git a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs index 96fbed59..b9a55efa 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs @@ -23,7 +23,7 @@ namespace Seq.Forwarder.Cli.Commands { - [Command("forwarder", "uninstall", "Uninstall the Windows service")] + [Command("forwarder", "uninstall", "Uninstall the forwarder Windows service")] class UninstallCommand : Command { protected override Task Run() From d41378ecc0fce5d552acfcaa64c9847df2f05495 Mon Sep 17 00:00:00 2001 From: KodrAus Date: Thu, 29 Feb 2024 16:40:01 +1000 Subject: [PATCH 17/51] fix build on Windows --- .../Cli/Commands/Forwarder/InstallCommand.cs | 16 +++++++++------- src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs | 4 +++- .../Cli/Commands/Forwarder/UninstallCommand.cs | 1 + .../DpapiMachineScopeDataProtection.cs | 1 + .../Cryptography/StringDataProtector.cs | 4 +++- .../ServiceProcess/SeqForwarderWindowsService.cs | 2 +- .../Forwarder/Util/ExecutionEnvironment.cs | 4 +++- .../Forwarder/Util/ServiceConfiguration.cs | 1 + 8 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs index 3eeda939..a598b87f 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs @@ -22,12 +22,14 @@ using System.ServiceProcess; using System.Threading.Tasks; using Seq.Forwarder.Cli.Features; -using Seq.Forwarder.Config; using Seq.Forwarder.ServiceProcess; using Seq.Forwarder.Util; using SeqCli; using SeqCli.Cli; using SeqCli.Cli.Features; +using SeqCli.Config; +using SeqCli.Config.Forwarder; +using SeqCli.Forwarder.Util; // ReSharper disable once ClassNeverInstantiated.Global @@ -181,12 +183,12 @@ void Install() throw new ArgumentException("Seq requires a local (or SAN) storage location; network shares are not supported."); Console.WriteLine($"Updating the configuration in {_storagePath.ConfigFilePath}..."); - var config = SeqForwarderConfig.ReadOrInit(_storagePath.ConfigFilePath); + var config = SeqCliConfig.Read(); if (!string.IsNullOrEmpty(_listenUri.ListenUri)) { - config.Api.ListenUri = _listenUri.ListenUri; - SeqForwarderConfig.Write(_storagePath.ConfigFilePath, config); + config.Forwarder.Api.ListenUri = _listenUri.ListenUri; + SeqCliConfig.Write(config); } if (_serviceCredentials.IsUsernameSpecified) @@ -203,10 +205,10 @@ void Install() Console.WriteLine($"Granting {ServiceUsername} rights to {_storagePath.StorageRootPath}..."); GiveFullControl(_storagePath.StorageRootPath); - Console.WriteLine($"Granting {ServiceUsername} rights to {config.Diagnostics.InternalLogPath}..."); - GiveFullControl(config.Diagnostics.InternalLogPath); + Console.WriteLine($"Granting {ServiceUsername} rights to {config.Forwarder.Diagnostics.InternalLogPath}..."); + GiveFullControl(config.Forwarder.Diagnostics.InternalLogPath); - var listenUri = MakeListenUriReservationPattern(config.Api.ListenUri); + var listenUri = MakeListenUriReservationPattern(config.Forwarder.Api.ListenUri); Console.WriteLine($"Adding URL reservation at {listenUri} for {ServiceUsername}..."); var netshResult = CaptiveProcess.Run("netsh", $"http add urlacl url={listenUri} user=\"{ServiceUsername}\"", Console.WriteLine, Console.WriteLine); if (netshResult != 0) diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index 3c94225e..df5e1b84 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -16,11 +16,13 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net; +using System.Security.Cryptography.X509Certificates; using System.Threading.Tasks; using Autofac; using Autofac.Extensions.DependencyInjection; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Hosting; +using Seq.Forwarder.ServiceProcess; using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Config.Forwarder; @@ -169,7 +171,7 @@ static int RunService(ServerService service) { #if WINDOWS System.ServiceProcess.ServiceBase.Run([ - new ServiceProcess.SeqForwarderWindowsService(service) + new SeqForwarderWindowsService(service) ]); return 0; #else diff --git a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs index b9a55efa..c823a04c 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs @@ -20,6 +20,7 @@ using Seq.Forwarder.ServiceProcess; using Seq.Forwarder.Util; using SeqCli.Cli; +using SeqCli.Forwarder.Util; namespace Seq.Forwarder.Cli.Commands { diff --git a/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs b/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs index 635bb415..2eb74421 100644 --- a/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs +++ b/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs @@ -18,6 +18,7 @@ using System.Diagnostics.CodeAnalysis; using System.Security.Cryptography; using System.Text; +using SeqCli.Forwarder.Cryptography; namespace Seq.Forwarder.Cryptography { diff --git a/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs b/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs index e35ef0b7..97c46021 100644 --- a/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs +++ b/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs @@ -1,4 +1,6 @@ -namespace SeqCli.Forwarder.Cryptography; +using Seq.Forwarder.Cryptography; + +namespace SeqCli.Forwarder.Cryptography; static class StringDataProtector { diff --git a/src/SeqCli/Forwarder/ServiceProcess/SeqForwarderWindowsService.cs b/src/SeqCli/Forwarder/ServiceProcess/SeqForwarderWindowsService.cs index a77efec1..ec944c8b 100644 --- a/src/SeqCli/Forwarder/ServiceProcess/SeqForwarderWindowsService.cs +++ b/src/SeqCli/Forwarder/ServiceProcess/SeqForwarderWindowsService.cs @@ -17,7 +17,7 @@ using System.Diagnostics.CodeAnalysis; using System.Net; using System.ServiceProcess; -using Seq.Forwarder.Web.Host; +using SeqCli.Forwarder.Web.Host; namespace Seq.Forwarder.ServiceProcess { diff --git a/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs b/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs index e0449984..300d77b5 100644 --- a/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs +++ b/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs @@ -1,4 +1,6 @@ -namespace SeqCli.Forwarder.Util; +using Seq.Forwarder.Util; + +namespace SeqCli.Forwarder.Util; static class ExecutionEnvironment { diff --git a/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs b/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs index b15c4775..f1739326 100644 --- a/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs +++ b/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs @@ -20,6 +20,7 @@ using System.Linq; using System.ServiceProcess; using System.Text; +using SeqCli.Forwarder.Util; namespace Seq.Forwarder.Util { From aa8d6710710cbaae04c90585f1d34d4bd320e43c Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 1 Mar 2024 10:10:28 +1000 Subject: [PATCH 18/51] build on linux --- src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs | 1 - src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs | 4 +++- src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index df5e1b84..62b79989 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -22,7 +22,6 @@ using Autofac.Extensions.DependencyInjection; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Hosting; -using Seq.Forwarder.ServiceProcess; using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Config.Forwarder; diff --git a/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs b/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs index 97c46021..6b67c890 100644 --- a/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs +++ b/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs @@ -1,4 +1,6 @@ -using Seq.Forwarder.Cryptography; +#if WINDOWS +using Seq.Forwarder.Cryptography; +#endif namespace SeqCli.Forwarder.Cryptography; diff --git a/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs b/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs index 300d77b5..40b47071 100644 --- a/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs +++ b/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs @@ -1,4 +1,6 @@ -using Seq.Forwarder.Util; +#if WINDOWS +using Seq.Forwarder.Util; +#endif namespace SeqCli.Forwarder.Util; From e448300391e7416ec7c32dafab9256e7cae42abc Mon Sep 17 00:00:00 2001 From: KodrAus Date: Fri, 1 Mar 2024 10:42:04 +1000 Subject: [PATCH 19/51] remove forwarder data protection in favor of seqcli --- .../Cli/Commands/Profile/CreateCommand.cs | 4 +- src/SeqCli/Config/ConnectionConfig.cs | 51 ++++++++----------- src/SeqCli/Config/SeqCliConfig.cs | 2 +- .../Config/SeqCliEncryptionProviderConfig.cs | 16 ++++++ src/SeqCli/Connection/SeqConnectionFactory.cs | 5 +- ...Encryption.cs => ExternalDataProtector.cs} | 4 +- .../{IEncryption.cs => IDataProtector.cs} | 2 +- ...ncryption.cs => PlaintextDataProtector.cs} | 2 +- ...ption.cs => WindowsNativeDataProtector.cs} | 2 +- .../DpapiMachineScopeDataProtection.cs | 49 ------------------ .../Cryptography/IStringDataProtector.cs | 7 --- .../Cryptography/StringDataProtector.cs | 15 ------ .../Cryptography/UnprotectedStringData.cs | 21 -------- .../Multiplexing/ActiveLogBufferMap.cs | 17 ++++--- .../Multiplexing/ActiveLogBufferMapTests.cs | 4 +- 15 files changed, 61 insertions(+), 140 deletions(-) rename src/SeqCli/Encryptor/{ExternalEncryption.cs => ExternalDataProtector.cs} (96%) rename src/SeqCli/Encryptor/{IEncryption.cs => IDataProtector.cs} (79%) rename src/SeqCli/Encryptor/{PlaintextEncryption.cs => PlaintextDataProtector.cs} (81%) rename src/SeqCli/Encryptor/{WindowsNativeEncryption.cs => WindowsNativeDataProtector.cs} (94%) delete mode 100644 src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs delete mode 100644 src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs delete mode 100644 src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs delete mode 100644 src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs diff --git a/src/SeqCli/Cli/Commands/Profile/CreateCommand.cs b/src/SeqCli/Cli/Commands/Profile/CreateCommand.cs index d6769739..866ee9f7 100644 --- a/src/SeqCli/Cli/Commands/Profile/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Profile/CreateCommand.cs @@ -49,7 +49,9 @@ int RunSync() try { var config = SeqCliConfig.Read(); - config.Profiles[_name] = new ConnectionConfig { ServerUrl = _url, ApiKey = _apiKey }; + var connectionConfig = new ConnectionConfig { ServerUrl = _url }; + connectionConfig.EncodeApiKey(_apiKey, config.Encryption.DataProtector()); + config.Profiles[_name] = connectionConfig; SeqCliConfig.Write(config); return 0; } diff --git a/src/SeqCli/Config/ConnectionConfig.cs b/src/SeqCli/Config/ConnectionConfig.cs index 821b6ad6..d2f31289 100644 --- a/src/SeqCli/Config/ConnectionConfig.cs +++ b/src/SeqCli/Config/ConnectionConfig.cs @@ -13,8 +13,9 @@ // limitations under the License. using System; +using System.Text; using Newtonsoft.Json; -using SeqCli.Forwarder.Cryptography; +using SeqCli.Encryptor; using SeqCli.Util; namespace SeqCli.Config; @@ -23,47 +24,37 @@ public class ConnectionConfig { const string ProtectedDataPrefix = "pd."; + static readonly Encoding ProtectedDataEncoding = new UTF8Encoding(false); + public string ServerUrl { get; set; } = "http://localhost:5341"; [JsonProperty("apiKey")] public string? EncodedApiKey { get; set; } - [JsonIgnore] - public string? ApiKey + public string? DecodeApiKey(IDataProtector dataProtector) { - get - { - if (string.IsNullOrWhiteSpace(EncodedApiKey)) - return null; - - if (!OperatingSystem.IsWindows()) - return EncodedApiKey; + if (string.IsNullOrWhiteSpace(EncodedApiKey)) + return null; + + if (!EncodedApiKey.StartsWith(ProtectedDataPrefix)) + return EncodedApiKey; - if (!EncodedApiKey.StartsWith(ProtectedDataPrefix)) - return EncodedApiKey; + return ProtectedDataEncoding.GetString(dataProtector.Decrypt(Convert.FromBase64String(EncodedApiKey[ProtectedDataPrefix.Length..]))); + } - return UserScopeDataProtection.Unprotect(EncodedApiKey.Substring(ProtectedDataPrefix.Length)); - } - set + public void EncodeApiKey(string? apiKey, IDataProtector dataProtector) + { + if (apiKey == null) { - if (string.IsNullOrWhiteSpace(value)) - { - EncodedApiKey = null; - return; - } - - if (OperatingSystem.IsWindows()) - EncodedApiKey = $"{ProtectedDataPrefix}{UserScopeDataProtection.Protect(value)}"; - else - EncodedApiKey = value; + EncodedApiKey = null; + return; } - } - public string? GetApiKey(IStringDataProtector dataProtector) - { - throw new NotImplementedException(); + var encoded = dataProtector.Encrypt(ProtectedDataEncoding.GetBytes(apiKey)); + + EncodedApiKey = $"{ProtectedDataPrefix}{Convert.ToBase64String(encoded)}"; } - + public uint? PooledConnectionLifetimeMilliseconds { get; set; } = null; public ulong EventBodyLimitBytes { get; set; } = 256 * 1024; public ulong PayloadLimitBytes { get; set; } = 10 * 1024 * 1024; diff --git a/src/SeqCli/Config/SeqCliConfig.cs b/src/SeqCli/Config/SeqCliConfig.cs index 88ac1b08..752476b4 100644 --- a/src/SeqCli/Config/SeqCliConfig.cs +++ b/src/SeqCli/Config/SeqCliConfig.cs @@ -57,7 +57,7 @@ public static void Write(SeqCliConfig data) public ConnectionConfig Connection { get; set; } = new(); public OutputConfig Output { get; set; } = new(); public ForwarderConfig Forwarder { get; set; } = new(); - public SeqCliEncryptionProviderConfig EncryptionProviderProvider { get; set; } = new SeqCliEncryptionProviderConfig(); + public SeqCliEncryptionProviderConfig Encryption { get; set; } = new SeqCliEncryptionProviderConfig(); public Dictionary Profiles { get; } = new(StringComparer.OrdinalIgnoreCase); } \ No newline at end of file diff --git a/src/SeqCli/Config/SeqCliEncryptionProviderConfig.cs b/src/SeqCli/Config/SeqCliEncryptionProviderConfig.cs index c7818e18..d94750aa 100644 --- a/src/SeqCli/Config/SeqCliEncryptionProviderConfig.cs +++ b/src/SeqCli/Config/SeqCliEncryptionProviderConfig.cs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +using SeqCli.Encryptor; + namespace SeqCli.Config; public class SeqCliEncryptionProviderConfig @@ -21,4 +23,18 @@ public class SeqCliEncryptionProviderConfig public string? Decryptor { get; set; } public string? DecryptorArgs { get; set; } + + public IDataProtector DataProtector() + { +#if WINDOWS + return new WindowsNativeDataProtector(); +#else + if (!string.IsNullOrWhiteSpace(Encryptor) && !string.IsNullOrWhiteSpace(Decryptor)) + { + return new ExternalDataProtector(this); + } + + return new PlaintextDataProtector(); +#endif + } } \ No newline at end of file diff --git a/src/SeqCli/Connection/SeqConnectionFactory.cs b/src/SeqCli/Connection/SeqConnectionFactory.cs index 08ef3cfb..05a82d73 100644 --- a/src/SeqCli/Connection/SeqConnectionFactory.cs +++ b/src/SeqCli/Connection/SeqConnectionFactory.cs @@ -16,6 +16,7 @@ using Seq.Api; using SeqCli.Cli.Features; using SeqCli.Config; +using SeqCli.Encryptor; namespace SeqCli.Connection; @@ -50,12 +51,12 @@ public SeqConnection Connect(ConnectionFeature connection) throw new ArgumentException($"A profile named `{connection.ProfileName}` was not found; see `seqcli profile list` for available profiles."); url = profile.ServerUrl; - apiKey = profile.ApiKey; + apiKey = profile.DecodeApiKey(_config.Encryption.DataProtector()); } else { url = _config.Connection.ServerUrl; - apiKey = connection.IsApiKeySpecified ? connection.ApiKey : _config.Connection.ApiKey; + apiKey = connection.IsApiKeySpecified ? connection.ApiKey : _config.Connection.DecodeApiKey(_config.Encryption.DataProtector()); } return (url, apiKey); diff --git a/src/SeqCli/Encryptor/ExternalEncryption.cs b/src/SeqCli/Encryptor/ExternalDataProtector.cs similarity index 96% rename from src/SeqCli/Encryptor/ExternalEncryption.cs rename to src/SeqCli/Encryptor/ExternalDataProtector.cs index b9db753f..3444b37b 100644 --- a/src/SeqCli/Encryptor/ExternalEncryption.cs +++ b/src/SeqCli/Encryptor/ExternalDataProtector.cs @@ -7,9 +7,9 @@ namespace SeqCli.Encryptor; -public class ExternalEncryption : IEncryption +public class ExternalDataProtector : IDataProtector { - public ExternalEncryption(SeqCliEncryptionProviderConfig providerConfig) + public ExternalDataProtector(SeqCliEncryptionProviderConfig providerConfig) { _encryptor = providerConfig.Encryptor!; _encryptorArgs = providerConfig.EncryptorArgs; diff --git a/src/SeqCli/Encryptor/IEncryption.cs b/src/SeqCli/Encryptor/IDataProtector.cs similarity index 79% rename from src/SeqCli/Encryptor/IEncryption.cs rename to src/SeqCli/Encryptor/IDataProtector.cs index 0294fa82..06db6d34 100644 --- a/src/SeqCli/Encryptor/IEncryption.cs +++ b/src/SeqCli/Encryptor/IDataProtector.cs @@ -1,6 +1,6 @@ namespace SeqCli.Encryptor; -public interface IEncryption +public interface IDataProtector { public byte[] Encrypt(byte[] unencrypted); public byte[] Decrypt(byte[] encrypted); diff --git a/src/SeqCli/Encryptor/PlaintextEncryption.cs b/src/SeqCli/Encryptor/PlaintextDataProtector.cs similarity index 81% rename from src/SeqCli/Encryptor/PlaintextEncryption.cs rename to src/SeqCli/Encryptor/PlaintextDataProtector.cs index 53a8df3e..e464002d 100644 --- a/src/SeqCli/Encryptor/PlaintextEncryption.cs +++ b/src/SeqCli/Encryptor/PlaintextDataProtector.cs @@ -1,6 +1,6 @@ namespace SeqCli.Encryptor; -class PlaintextEncryption : IEncryption +class PlaintextDataProtector : IDataProtector { public byte[] Encrypt(byte[] unencrypted) { diff --git a/src/SeqCli/Encryptor/WindowsNativeEncryption.cs b/src/SeqCli/Encryptor/WindowsNativeDataProtector.cs similarity index 94% rename from src/SeqCli/Encryptor/WindowsNativeEncryption.cs rename to src/SeqCli/Encryptor/WindowsNativeDataProtector.cs index 323a82df..203d0f20 100644 --- a/src/SeqCli/Encryptor/WindowsNativeEncryption.cs +++ b/src/SeqCli/Encryptor/WindowsNativeDataProtector.cs @@ -5,7 +5,7 @@ namespace SeqCli.Encryptor; -public class WindowsNativeEncryption : IEncryption +public class WindowsNativeDataProtector : IDataProtector { public byte[] Encrypt(byte[] unencrypted) { diff --git a/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs b/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs deleted file mode 100644 index 2eb74421..00000000 --- a/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright © Datalust Pty Ltd and Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#if WINDOWS - -using System; -using System.Diagnostics.CodeAnalysis; -using System.Security.Cryptography; -using System.Text; -using SeqCli.Forwarder.Cryptography; - -namespace Seq.Forwarder.Cryptography -{ - [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] - class DpapiMachineScopeDataProtect : IStringDataProtector - { - public string Unprotect(string @protected) - { - var parts = @protected.Split(new[] { '$' }, StringSplitOptions.RemoveEmptyEntries); - if (parts.Length != 2) - throw new InvalidOperationException("Encoded data format is invalid."); - - var bytes = Convert.FromBase64String(parts[0]); - var salt = Convert.FromBase64String(parts[1]); - var decoded = ProtectedData.Unprotect(bytes, salt, DataProtectionScope.LocalMachine); - return Encoding.UTF8.GetString(decoded); - } - - public string Protect(string value) - { - var salt = RandomNumberGenerator.GetBytes(16); - var bytes = ProtectedData.Protect(Encoding.UTF8.GetBytes(value), salt, DataProtectionScope.LocalMachine); - return $"{Convert.ToBase64String(bytes)}${Convert.ToBase64String(salt)}"; - } - } -} - -#endif diff --git a/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs b/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs deleted file mode 100644 index cdc930c1..00000000 --- a/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace SeqCli.Forwarder.Cryptography; - -public interface IStringDataProtector -{ - string Protect(string value); - string Unprotect(string @protected); -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs b/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs deleted file mode 100644 index 97c46021..00000000 --- a/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs +++ /dev/null @@ -1,15 +0,0 @@ -using Seq.Forwarder.Cryptography; - -namespace SeqCli.Forwarder.Cryptography; - -static class StringDataProtector -{ - public static IStringDataProtector CreatePlatformDefault() - { -#if WINDOWS - return new DpapiMachineScopeDataProtect(); -#else - return new UnprotectedStringData(); -#endif - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs b/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs deleted file mode 100644 index 6148081e..00000000 --- a/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs +++ /dev/null @@ -1,21 +0,0 @@ -#if !WINDOWS - -using Serilog; - -namespace SeqCli.Forwarder.Cryptography; - -public class UnprotectedStringData : IStringDataProtector -{ - public string Protect(string value) - { - Log.Warning("Data protection is not available on this platform; sensitive values will be stored in plain text"); - return value; - } - - public string Unprotect(string @protected) - { - return @protected; - } -} - -#endif diff --git a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs index bcd09c65..a18ff2ac 100644 --- a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs +++ b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs @@ -16,9 +16,10 @@ using System.Collections.Generic; using System.IO; using System.Net; +using System.Text; using SeqCli.Config; using SeqCli.Config.Forwarder; -using SeqCli.Forwarder.Cryptography; +using SeqCli.Encryptor; using SeqCli.Forwarder.Storage; using SeqCli.Forwarder.Web; using Serilog; @@ -29,10 +30,12 @@ public class ActiveLogBufferMap : IDisposable { const string DataFileName = "data.mdb", LockFileName = "lock.mdb", ApiKeyFileName = ".apikey"; + static Encoding ApiKeyEncoding = new UTF8Encoding(false); + readonly ulong _bufferSizeBytes; readonly ConnectionConfig _connectionConfig; readonly ILogShipperFactory _shipperFactory; - readonly IStringDataProtector _dataProtector; + readonly IDataProtector _dataProtector; readonly string _bufferPath; readonly ILogger _log = Log.ForContext(); @@ -46,7 +49,7 @@ public ActiveLogBufferMap( ForwarderStorageConfig storageConfig, ConnectionConfig outputConfig, ILogShipperFactory logShipperFactory, - IStringDataProtector dataProtector) + IDataProtector dataProtector) { _bufferSizeBytes = storageConfig.BufferSizeBytes; _connectionConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); @@ -86,7 +89,7 @@ public void Load() } else { - _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _connectionConfig.GetApiKey(_dataProtector))); + _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _connectionConfig.DecodeApiKey(_dataProtector))); } } @@ -100,7 +103,7 @@ public void Load() } _log.Information("Loading an API-key specific buffer in {Path}", subfolder); - var apiKey = _dataProtector.Unprotect(File.ReadAllText(encodedApiKeyFilePath)); + var apiKey = ApiKeyEncoding.GetString(_dataProtector.Decrypt(File.ReadAllBytes(encodedApiKeyFilePath))); var buffer = new LogBuffer(subfolder, _bufferSizeBytes); if (buffer.Peek(0).Length == 0) @@ -159,7 +162,7 @@ public LogBuffer GetLogBuffer(string? apiKey) { _log.Information("Creating a new default log buffer in {Path}", _bufferPath); var buffer = new LogBuffer(_bufferPath, _bufferSizeBytes); - _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _connectionConfig.GetApiKey(_dataProtector))); + _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _connectionConfig.DecodeApiKey(_dataProtector))); _noApiKeyLogBuffer.Shipper.Start(); } return _noApiKeyLogBuffer.Buffer; @@ -171,7 +174,7 @@ public LogBuffer GetLogBuffer(string? apiKey) var subfolder = Path.Combine(_bufferPath, Guid.NewGuid().ToString("n")); _log.Information("Creating a new API key-specific log buffer in {Path}", subfolder); Directory.CreateDirectory(subfolder); - File.WriteAllText(Path.Combine(subfolder, ".apikey"), _dataProtector.Protect(apiKey)); + File.WriteAllBytes(Path.Combine(subfolder, ".apikey"), _dataProtector.Encrypt(ApiKeyEncoding.GetBytes(apiKey))); var newBuffer = new LogBuffer(subfolder, _bufferSizeBytes); var newActiveBuffer = new ActiveLogBuffer(newBuffer, _shipperFactory.Create(newBuffer, apiKey)); _buffersByApiKey.Add(apiKey, newActiveBuffer); diff --git a/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs index 11db09d1..6634f929 100644 --- a/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs +++ b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs @@ -1,7 +1,7 @@ using System.IO; using System.Linq; using SeqCli.Config; -using SeqCli.Forwarder.Cryptography; +using SeqCli.Encryptor; using SeqCli.Forwarder.Multiplexing; using SeqCli.Tests.Support; using Xunit; @@ -74,7 +74,7 @@ public void EntriesSurviveReloads() static ActiveLogBufferMap CreateActiveLogBufferMap(TempFolder tmp) { var config = new SeqCliConfig(); - var map = new ActiveLogBufferMap(tmp.Path, config.Forwarder.Storage, config.Connection, new InertLogShipperFactory(), StringDataProtector.CreatePlatformDefault()); + var map = new ActiveLogBufferMap(tmp.Path, config.Forwarder.Storage, config.Connection, new InertLogShipperFactory(), new PlaintextDataProtector()); map.Load(); return map; } From d670753abdbed574987f76b0f6db47be1e2852fa Mon Sep 17 00:00:00 2001 From: KodrAus Date: Fri, 1 Mar 2024 11:00:27 +1000 Subject: [PATCH 20/51] get forwarder run command executing --- src/SeqCli/Forwarder/ForwarderModule.cs | 15 +++++---------- .../Multiplexing/ActiveLogBufferMap.cs | 18 ++++++++---------- .../Multiplexing/HttpLogShipperFactory.cs | 6 ++++-- .../Forwarder/Web/Api/IngestionController.cs | 2 +- src/SeqCli/SeqCliModule.cs | 2 ++ .../Multiplexing/ActiveLogBufferMapTests.cs | 3 +-- 6 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 2a5005ab..019afd4f 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -17,7 +17,7 @@ using System.Threading; using Autofac; using SeqCli.Config; -using SeqCli.Forwarder.Cryptography; +using SeqCli.Encryptor; using SeqCli.Forwarder.Multiplexing; using SeqCli.Forwarder.Web.Host; @@ -46,8 +46,8 @@ protected override void Load(ContainerBuilder builder) builder.Register(c => { - var outputConfig = c.Resolve(); - var baseUri = outputConfig.ServerUrl; + var config = c.Resolve(); + var baseUri = config.Connection.ServerUrl; if (string.IsNullOrWhiteSpace(baseUri)) throw new ArgumentException("The destination Seq server URL must be configured in SeqForwarder.json."); @@ -58,13 +58,13 @@ protected override void Load(ContainerBuilder builder) // this expression, using an "or" operator. var hasSocketHandlerOption = - outputConfig.PooledConnectionLifetimeMilliseconds.HasValue; + config.Connection.PooledConnectionLifetimeMilliseconds.HasValue; if (hasSocketHandlerOption) { var httpMessageHandler = new SocketsHttpHandler { - PooledConnectionLifetime = outputConfig.PooledConnectionLifetimeMilliseconds.HasValue ? TimeSpan.FromMilliseconds(outputConfig.PooledConnectionLifetimeMilliseconds.Value) : Timeout.InfiniteTimeSpan, + PooledConnectionLifetime = config.Connection.PooledConnectionLifetimeMilliseconds.HasValue ? TimeSpan.FromMilliseconds(config.Connection.PooledConnectionLifetimeMilliseconds.Value) : Timeout.InfiniteTimeSpan, }; return new HttpClient(httpMessageHandler) { BaseAddress = new Uri(baseUri) }; @@ -74,11 +74,6 @@ protected override void Load(ContainerBuilder builder) }).SingleInstance(); - builder.RegisterInstance(StringDataProtector.CreatePlatformDefault()); - builder.RegisterInstance(_config); - builder.RegisterInstance(_config.Forwarder.Api); - builder.RegisterInstance(_config.Forwarder.Diagnostics); - builder.RegisterInstance(_config.Forwarder.Storage); } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs index a18ff2ac..92f96f2b 100644 --- a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs +++ b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs @@ -18,7 +18,6 @@ using System.Net; using System.Text; using SeqCli.Config; -using SeqCli.Config.Forwarder; using SeqCli.Encryptor; using SeqCli.Forwarder.Storage; using SeqCli.Forwarder.Web; @@ -26,11 +25,11 @@ namespace SeqCli.Forwarder.Multiplexing; -public class ActiveLogBufferMap : IDisposable +class ActiveLogBufferMap : IDisposable { const string DataFileName = "data.mdb", LockFileName = "lock.mdb", ApiKeyFileName = ".apikey"; - static Encoding ApiKeyEncoding = new UTF8Encoding(false); + static readonly Encoding ApiKeyEncoding = new UTF8Encoding(false); readonly ulong _bufferSizeBytes; readonly ConnectionConfig _connectionConfig; @@ -46,15 +45,14 @@ public class ActiveLogBufferMap : IDisposable public ActiveLogBufferMap( string bufferPath, - ForwarderStorageConfig storageConfig, - ConnectionConfig outputConfig, - ILogShipperFactory logShipperFactory, - IDataProtector dataProtector) + SeqCliConfig config, + ILogShipperFactory logShipperFactory) { - _bufferSizeBytes = storageConfig.BufferSizeBytes; - _connectionConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); + ArgumentNullException.ThrowIfNull(config, nameof(config)); + _bufferSizeBytes = config.Forwarder.Storage.BufferSizeBytes; + _connectionConfig = config.Connection; _shipperFactory = logShipperFactory ?? throw new ArgumentNullException(nameof(logShipperFactory)); - _dataProtector = dataProtector ?? throw new ArgumentNullException(nameof(dataProtector)); + _dataProtector = config.Encryption.DataProtector(); _bufferPath = bufferPath ?? throw new ArgumentNullException(nameof(bufferPath)); } diff --git a/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs index 7c95215d..3101421a 100644 --- a/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs +++ b/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs @@ -26,11 +26,13 @@ class HttpLogShipperFactory : ILogShipperFactory readonly ServerResponseProxy _serverResponseProxy; readonly ConnectionConfig _outputConfig; - public HttpLogShipperFactory(ServerResponseProxy serverResponseProxy, ConnectionConfig outputConfig, HttpClient outputHttpClient) + public HttpLogShipperFactory(SeqCliConfig config, ServerResponseProxy serverResponseProxy, HttpClient outputHttpClient) { + ArgumentNullException.ThrowIfNull(config, nameof(config)); + _outputHttpClient = outputHttpClient; _serverResponseProxy = serverResponseProxy ?? throw new ArgumentNullException(nameof(serverResponseProxy)); - _outputConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); + _outputConfig = config.Connection; } public LogShipper Create(LogBuffer logBuffer, string? apiKey) diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionController.cs b/src/SeqCli/Forwarder/Web/Api/IngestionController.cs index 047a8898..6510df77 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionController.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionController.cs @@ -32,7 +32,7 @@ namespace SeqCli.Forwarder.Web.Api; -public class IngestionController : Controller +class IngestionController : Controller { static readonly Encoding Encoding = new UTF8Encoding(false); const string ClefMediaType = "application/vnd.serilog.clef"; diff --git a/src/SeqCli/SeqCliModule.cs b/src/SeqCli/SeqCliModule.cs index 009d5172..658153bc 100644 --- a/src/SeqCli/SeqCliModule.cs +++ b/src/SeqCli/SeqCliModule.cs @@ -17,6 +17,7 @@ using SeqCli.Cli; using SeqCli.Config; using SeqCli.Connection; +using SeqCli.Encryptor; namespace SeqCli; @@ -32,5 +33,6 @@ protected override void Load(ContainerBuilder builder) builder.Register(c => SeqCliConfig.Read()).SingleInstance(); builder.Register(c => c.Resolve().Connection).SingleInstance(); builder.Register(c => c.Resolve().Output).SingleInstance(); + builder.Register(c => c.Resolve().Encryption.DataProtector()).As(); } } \ No newline at end of file diff --git a/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs index 6634f929..a4ca2484 100644 --- a/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs +++ b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs @@ -1,7 +1,6 @@ using System.IO; using System.Linq; using SeqCli.Config; -using SeqCli.Encryptor; using SeqCli.Forwarder.Multiplexing; using SeqCli.Tests.Support; using Xunit; @@ -74,7 +73,7 @@ public void EntriesSurviveReloads() static ActiveLogBufferMap CreateActiveLogBufferMap(TempFolder tmp) { var config = new SeqCliConfig(); - var map = new ActiveLogBufferMap(tmp.Path, config.Forwarder.Storage, config.Connection, new InertLogShipperFactory(), new PlaintextDataProtector()); + var map = new ActiveLogBufferMap(tmp.Path, config, new InertLogShipperFactory()); map.Load(); return map; } From 4caf14beba97abb4f9f2ac885ddad7ec99f261d4 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 1 Mar 2024 13:01:30 +1000 Subject: [PATCH 21/51] Switch to minimal API --- .../Cli/Commands/Forwarder/RunCommand.cs | 80 +++++++++---------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index 62b79989..d56cfe85 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -20,6 +20,7 @@ using System.Threading.Tasks; using Autofac; using Autofac.Extensions.DependencyInjection; +using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Hosting; using SeqCli.Cli.Features; @@ -93,54 +94,51 @@ protected override async Task Run(string[] unrecognized) try { ILifetimeScope? container = null; - using var host = new HostBuilder() - .UseSerilog() - .UseServiceProviderFactory(new AutofacServiceProviderFactory()) - .ConfigureContainer(builder => - { - builder.RegisterBuildCallback(ls => container = ls); - builder.RegisterModule(new ForwarderModule(_storagePath.BufferPath, config)); - }) - .ConfigureWebHostDefaults(web => - { - web.UseStartup(); - web.UseKestrel(options => - { - options.AddServerHeader = false; - options.AllowSynchronousIO = true; - }) - .ConfigureKestrel(options => - { - var apiListenUri = new Uri(listenUri); + var builder = WebApplication.CreateBuilder(); + builder.WebHost.UseKestrel(options => + { + options.AddServerHeader = false; + options.AllowSynchronousIO = true; + }).ConfigureKestrel((context, options) => + { + var apiListenUri = new Uri(listenUri); - var ipAddress = apiListenUri.HostNameType switch - { - UriHostNameType.Basic => IPAddress.Any, - UriHostNameType.Dns => IPAddress.Any, - UriHostNameType.IPv4 => IPAddress.Parse(apiListenUri.Host), - UriHostNameType.IPv6 => IPAddress.Parse(apiListenUri.Host), - _ => throw new NotSupportedException($"Listen URI type `{apiListenUri.HostNameType}` is not supported.") - }; + var ipAddress = apiListenUri.HostNameType switch + { + UriHostNameType.Basic => IPAddress.Any, + UriHostNameType.Dns => IPAddress.Any, + UriHostNameType.IPv4 => IPAddress.Parse(apiListenUri.Host), + UriHostNameType.IPv6 => IPAddress.Parse(apiListenUri.Host), + _ => throw new NotSupportedException($"Listen URI type `{apiListenUri.HostNameType}` is not supported.") + }; - if (apiListenUri.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase)) - { - options.Listen(ipAddress, apiListenUri.Port, listenOptions => - { + if (apiListenUri.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase)) + { + options.Listen(ipAddress, apiListenUri.Port, listenOptions => + { #if WINDOWS listenOptions.UseHttps(StoreName.My, apiListenUri.Host, location: StoreLocation.LocalMachine, allowInvalid: true); #else - listenOptions.UseHttps(); + listenOptions.UseHttps(); #endif - }); - } - else - { - options.Listen(ipAddress, apiListenUri.Port); - } - }); - }) - .Build(); + }); + } + else + { + options.Listen(ipAddress, apiListenUri.Port); + } + }); + + builder + .Host.UseSerilog() + .UseServiceProviderFactory(new AutofacServiceProviderFactory()) + .ConfigureContainer(builder => + { + builder.RegisterBuildCallback(ls => container = ls); + builder.RegisterModule(new ForwarderModule(_storagePath.BufferPath, config)); + }); + using var host = builder.Build(); if (container == null) throw new Exception("Host did not build container."); From 4c4f90db144d2e35e61b4e48dc392397eb99316e Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 1 Mar 2024 13:05:42 +1000 Subject: [PATCH 22/51] Fix build --- src/SeqCli/Cli/Features/StoragePathFeature.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/SeqCli/Cli/Features/StoragePathFeature.cs b/src/SeqCli/Cli/Features/StoragePathFeature.cs index 50523ca8..4283ddea 100644 --- a/src/SeqCli/Cli/Features/StoragePathFeature.cs +++ b/src/SeqCli/Cli/Features/StoragePathFeature.cs @@ -1,6 +1,5 @@ using System; using System.IO; -using SeqCli.Forwarder.ServiceProcess; namespace SeqCli.Cli.Features; From ef46a5fb9f1bc74895bb5e89b3e8700365a8a793 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 1 Mar 2024 13:37:12 +1000 Subject: [PATCH 23/51] ApiRoot endpoints --- .../Cli/Commands/Forwarder/RunCommand.cs | 9 +++- src/SeqCli/Forwarder/ForwarderModule.cs | 7 +++ .../Forwarder/Web/Api/ApiRootController.cs | 45 ++++++++----------- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index b310c845..5b5abd4f 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -28,11 +28,13 @@ using SeqCli.Config.Forwarder; using SeqCli.Forwarder; using SeqCli.Forwarder.Util; +using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; using Serilog; using Serilog.Core; using Serilog.Events; using Serilog.Formatting.Compact; +using Serilog.Formatting.Display; #if WINDOWS using SeqCli.Forwarder.ServiceProcess; @@ -142,10 +144,13 @@ protected override async Task Run(string[] unrecognized) builder.RegisterBuildCallback(ls => container = ls); builder.RegisterModule(new ForwarderModule(_storagePath.BufferPath, config)); }); + using var host = builder.Build(); - + if (container == null) throw new Exception("Host did not build container."); - + + ApiRoot.Map(host, container.Resolve()); + var service = container.Resolve( new TypedParameter(typeof(IHost), host), new NamedParameter("listenUri", listenUri)); diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 019afd4f..8dbc69b2 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -19,7 +19,9 @@ using SeqCli.Config; using SeqCli.Encryptor; using SeqCli.Forwarder.Multiplexing; +using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; +using Serilog.Formatting.Display; namespace SeqCli.Forwarder; @@ -43,6 +45,11 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType().As(); builder.RegisterType().SingleInstance(); + builder.RegisterType(); + builder.RegisterInstance(new MessageTemplateTextFormatter( + "[{Timestamp:o} {Level:u3}] {Message}{NewLine}" + (_config.Forwarder.Diagnostics.IngestionLogShowDetail + ? "" + : "Client IP address: {ClientHostIP}{NewLine}First {StartToLog} characters of payload: {DocumentStart:l}{NewLine}{Exception}{NewLine}"))).SingleInstance(); builder.Register(c => { diff --git a/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs b/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs index 9747a8d2..3d6a2d70 100644 --- a/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs +++ b/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs @@ -14,43 +14,36 @@ using System.IO; using System.Text; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Hosting; using SeqCli.Config.Forwarder; using SeqCli.Forwarder.Diagnostics; using Serilog.Formatting.Display; namespace SeqCli.Forwarder.Web.Api; -public class ApiRootController : Controller -{ +public class ApiRoot +{ static readonly Encoding Encoding = new UTF8Encoding(false); - readonly MessageTemplateTextFormatter _ingestionLogFormatter; - public ApiRootController(ForwarderDiagnosticConfig diagnosticConfig) + public static void Map(WebApplication app, MessageTemplateTextFormatter formatter) { - var template = "[{Timestamp:o} {Level:u3}] {Message}{NewLine}"; - if (diagnosticConfig.IngestionLogShowDetail) - template += "Client IP address: {ClientHostIP}{NewLine}First {StartToLog} characters of payload: {DocumentStart:l}{NewLine}{Exception}{NewLine}"; - - _ingestionLogFormatter = new MessageTemplateTextFormatter(template); - } - - [HttpGet, Route("")] - public IActionResult Index() - { - var events = IngestionLog.Read(); - using var log = new StringWriter(); - foreach (var logEvent in events) + app.MapGet("/", () => { - _ingestionLogFormatter.Format(logEvent, log); - } + var events = IngestionLog.Read(); + using var log = new StringWriter(); + foreach (var logEvent in events) + { + formatter.Format(logEvent, log); + } - return Content(log.ToString(), "text/plain", Encoding); - } + return Results.Content(log.ToString(), "text/plain", Encoding); + }); - [HttpGet, Route("api")] - public IActionResult Resources() - { - return Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Encoding); - } + app.MapGet("/api", + () => Results.Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Encoding)); + + } } \ No newline at end of file From cea428d1d87f89cbc07a3b08e1ba58c52c348567 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 1 Mar 2024 15:13:08 +1000 Subject: [PATCH 24/51] Remove MVC --- .../Cli/Commands/Forwarder/RunCommand.cs | 9 +- src/SeqCli/Forwarder/ForwarderModule.cs | 5 +- ...iRootController.cs => ApiRootEndpoints.cs} | 23 +- src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs | 8 + ...ionController.cs => IngestionEndpoints.cs} | 249 +++++++++--------- src/SeqCli/Forwarder/Web/Host/Startup.cs | 1 - 6 files changed, 156 insertions(+), 139 deletions(-) rename src/SeqCli/Forwarder/Web/Api/{ApiRootController.cs => ApiRootEndpoints.cs} (74%) create mode 100644 src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs rename src/SeqCli/Forwarder/Web/Api/{IngestionController.cs => IngestionEndpoints.cs} (57%) diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index 5b5abd4f..4c4cfa00 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -13,6 +13,8 @@ // limitations under the License. using System; +using System.Collections; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net; @@ -148,8 +150,11 @@ protected override async Task Run(string[] unrecognized) using var host = builder.Build(); if (container == null) throw new Exception("Host did not build container."); - - ApiRoot.Map(host, container.Resolve()); + + foreach (var mapper in container.Resolve>()) + { + mapper.Map(host); + } var service = container.Resolve( new TypedParameter(typeof(IHost), host), diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 8dbc69b2..a0df187d 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -17,7 +17,6 @@ using System.Threading; using Autofac; using SeqCli.Config; -using SeqCli.Encryptor; using SeqCli.Forwarder.Multiplexing; using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; @@ -45,7 +44,9 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType().As(); builder.RegisterType().SingleInstance(); - builder.RegisterType(); + builder.RegisterType().As(); + builder.RegisterType().As(); + builder.RegisterInstance(_config.Connection); builder.RegisterInstance(new MessageTemplateTextFormatter( "[{Timestamp:o} {Level:u3}] {Message}{NewLine}" + (_config.Forwarder.Diagnostics.IngestionLogShowDetail ? "" diff --git a/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs b/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs similarity index 74% rename from src/SeqCli/Forwarder/Web/Api/ApiRootController.cs rename to src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs index 3d6a2d70..90015f1b 100644 --- a/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs +++ b/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs @@ -16,19 +16,22 @@ using System.Text; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Hosting; -using SeqCli.Config.Forwarder; using SeqCli.Forwarder.Diagnostics; using Serilog.Formatting.Display; namespace SeqCli.Forwarder.Web.Api; -public class ApiRoot -{ - static readonly Encoding Encoding = new UTF8Encoding(false); +class ApiRootEndpoints : IMapEndpoints +{ + readonly MessageTemplateTextFormatter _formatter; + readonly Encoding Utf8 = new UTF8Encoding(false); - public static void Map(WebApplication app, MessageTemplateTextFormatter formatter) + public ApiRootEndpoints(MessageTemplateTextFormatter formatter) + { + _formatter = formatter; + } + + public void Map(WebApplication app) { app.MapGet("/", () => { @@ -36,14 +39,14 @@ public static void Map(WebApplication app, MessageTemplateTextFormatter formatte using var log = new StringWriter(); foreach (var logEvent in events) { - formatter.Format(logEvent, log); + _formatter.Format(logEvent, log); } - return Results.Content(log.ToString(), "text/plain", Encoding); + return Results.Content(log.ToString(), "text/plain", Utf8); }); app.MapGet("/api", - () => Results.Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Encoding)); + () => Results.Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Utf8)); } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs new file mode 100644 index 00000000..b5c53b1b --- /dev/null +++ b/src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs @@ -0,0 +1,8 @@ +using Microsoft.AspNetCore.Builder; + +namespace SeqCli.Forwarder.Web.Api; + +interface IMapEndpoints +{ + void Map(WebApplication app); +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionController.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs similarity index 57% rename from src/SeqCli/Forwarder/Web/Api/IngestionController.cs rename to src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index 6510df77..3a463c9d 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionController.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -19,8 +19,10 @@ using System.Net; using System.Text; using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.Features; -using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.Net.Http.Headers; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -32,86 +34,157 @@ namespace SeqCli.Forwarder.Web.Api; -class IngestionController : Controller +class IngestionEndpoints : IMapEndpoints { - static readonly Encoding Encoding = new UTF8Encoding(false); - const string ClefMediaType = "application/vnd.serilog.clef"; + static readonly Encoding Utf8 = new UTF8Encoding(false); readonly ActiveLogBufferMap _logBufferMap; - readonly ConnectionConfig _outputConfig; + readonly ConnectionConfig _connectionConfig; readonly ServerResponseProxy _serverResponseProxy; readonly JsonSerializer _rawSerializer = JsonSerializer.Create( new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); - public IngestionController(ActiveLogBufferMap logBufferMap, ConnectionConfig outputConfig, ServerResponseProxy serverResponseProxy) + public IngestionEndpoints( + ActiveLogBufferMap logBufferMap, + ServerResponseProxy serverResponseProxy, + ConnectionConfig connectionConfig) { _logBufferMap = logBufferMap; - _outputConfig = outputConfig; + _connectionConfig = connectionConfig; _serverResponseProxy = serverResponseProxy; } + + public void Map(WebApplication app) + { + app.MapGet("/api", + () => Results.Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Utf8)); - IPAddress ClientHostIP => Request.HttpContext.Connection.RemoteIpAddress!; + app.MapPost("api/events/raw", new Func>(async (context) => + { + var clef = DefaultedBoolQuery(context.Request, "clef"); - [HttpGet, Route("api/events/describe")] - public IActionResult Resources() - { - return Content("{\"Links\":{\"Raw\":\"/api/events/raw{?clef}\"}}", "application/json", Encoding); - } + if (clef) + return await IngestCompactFormat(context); - [HttpPost, Route("api/events/raw")] - public async Task Ingest() + var contentType = (string?) context.Request.Headers[HeaderNames.ContentType]; + var clefMediaType = "application/vnd.serilog.clef"; + + if (contentType != null && contentType.StartsWith(clefMediaType)) + return await IngestCompactFormat(context); + + return IngestRawFormat(context); + })); + } + + byte[][] EncodeRawEvents(ICollection events, IPAddress remoteIpAddress) { - var clef = DefaultedBoolQuery("clef"); + var encoded = new byte[events.Count][]; + var i = 0; + foreach (var e in events) + { + var s = e.ToString(Formatting.None); + var payload = Utf8.GetBytes(s); - if (clef) - return await IngestCompactFormat(); + if (payload.Length > (int) _connectionConfig.EventBodyLimitBytes) + { + IngestionLog.ForPayload(remoteIpAddress, s).Debug("An oversized event was dropped"); + + var jo = e as JObject; + // ReSharper disable SuspiciousTypeConversion.Global + var timestamp = (string?) (dynamic?) jo?.GetValue("Timestamp") ?? DateTime.UtcNow.ToString("o"); + var level = (string?) (dynamic?) jo?.GetValue("Level") ?? "Warning"; + + if (jo != null) + { + jo.Remove("Timestamp"); + jo.Remove("Level"); + } + + var startToLog = (int) Math.Min(_connectionConfig.EventBodyLimitBytes / 2, 1024); + var compactPrefix = e.ToString(Formatting.None).Substring(0, startToLog); + + encoded[i] = Utf8.GetBytes(JsonConvert.SerializeObject(new + { + Timestamp = timestamp, + MessageTemplate = "Seq Forwarder received and dropped an oversized event", + Level = level, + Properties = new + { + Partial = compactPrefix, + Environment.MachineName, + _connectionConfig.EventBodyLimitBytes, + PayloadBytes = payload.Length + } + })); + } + else + { + encoded[i] = payload; + } - var contentType = (string?) Request.Headers[HeaderNames.ContentType]; - if (contentType != null && contentType.StartsWith(ClefMediaType)) - return await IngestCompactFormat(); + i++; + } - return IngestRawFormat(); + return encoded; } + + static bool DefaultedBoolQuery(HttpRequest request, string queryParameterName) + { + var parameter = request.Query[queryParameterName]; + if (parameter.Count != 1) + return false; + + var value = (string?) parameter; - IActionResult IngestRawFormat() + if (value == "" && ( + request.QueryString.Value!.Contains($"&{queryParameterName}=") || + request.QueryString.Value.Contains($"?{queryParameterName}="))) + { + return false; + } + + return "true".Equals(value, StringComparison.OrdinalIgnoreCase) || value == "" || value == queryParameterName; + } + + IResult IngestRawFormat(HttpContext context) { // The compact format ingestion path works with async IO. - HttpContext.Features.Get()!.AllowSynchronousIO = true; - + context.Features.Get()!.AllowSynchronousIO = true; + JObject posted; try { - posted = _rawSerializer.Deserialize(new JsonTextReader(new StreamReader(Request.Body))) ?? + posted = _rawSerializer.Deserialize(new JsonTextReader(new StreamReader(context.Request.Body))) ?? throw new RequestProcessingException("Request body payload is JSON `null`."); } catch (Exception ex) { - IngestionLog.ForClient(ClientHostIP).Debug(ex,"Rejecting payload due to invalid JSON, request body could not be parsed"); + IngestionLog.ForClient(context.Connection.RemoteIpAddress!).Debug(ex,"Rejecting payload due to invalid JSON, request body could not be parsed"); throw new RequestProcessingException("Invalid raw event JSON, body could not be parsed."); } if (!(posted.TryGetValue("events", StringComparison.Ordinal, out var eventsToken) || posted.TryGetValue("Events", StringComparison.Ordinal, out eventsToken))) { - IngestionLog.ForClient(ClientHostIP).Debug("Rejecting payload due to invalid JSON structure"); + IngestionLog.ForClient(context.Connection.RemoteIpAddress!).Debug("Rejecting payload due to invalid JSON structure"); throw new RequestProcessingException("Invalid raw event JSON, body must contain an 'Events' array."); } if (!(eventsToken is JArray events)) { - IngestionLog.ForClient(ClientHostIP).Debug("Rejecting payload due to invalid Events property structure"); + IngestionLog.ForClient(context.Connection.RemoteIpAddress!).Debug("Rejecting payload due to invalid Events property structure"); throw new RequestProcessingException("Invalid raw event JSON, the 'Events' property must be an array."); } - var encoded = EncodeRawEvents(events); - return Enqueue(encoded); + var encoded = EncodeRawEvents(events, context.Connection.RemoteIpAddress!); + return Enqueue(context.Request, encoded); } - - async Task IngestCompactFormat() + + async Task IngestCompactFormat(HttpContext context) { var rawFormat = new List(); - var reader = new StreamReader(Request.Body); + var reader = new StreamReader(context.Request.Body); var line = await reader.ReadLineAsync(); var lineNumber = 1; @@ -128,13 +201,13 @@ async Task IngestCompactFormat() } catch (Exception ex) { - IngestionLog.ForPayload(ClientHostIP, line).Debug(ex, "Rejecting CLEF payload due to invalid JSON, item could not be parsed"); + IngestionLog.ForPayload(context.Connection.RemoteIpAddress!, line).Debug(ex, "Rejecting CLEF payload due to invalid JSON, item could not be parsed"); throw new RequestProcessingException($"Invalid raw event JSON, item on line {lineNumber} could not be parsed."); } if (!EventSchema.FromClefFormat(lineNumber, item, out var evt, out var err)) { - IngestionLog.ForPayload(ClientHostIP, line).Debug("Rejecting CLEF payload due to invalid event JSON structure: {NormalizationError}", err); + IngestionLog.ForPayload(context.Connection.RemoteIpAddress!, line).Debug("Rejecting CLEF payload due to invalid event JSON structure: {NormalizationError}", err); throw new RequestProcessingException(err); } @@ -145,101 +218,29 @@ async Task IngestCompactFormat() ++lineNumber; } - var encoded = EncodeRawEvents(rawFormat); - return Enqueue(encoded); + return Enqueue( + context.Request, + EncodeRawEvents(rawFormat, context.Connection.RemoteIpAddress!)); } - - byte[][] EncodeRawEvents(ICollection events) + + ContentHttpResult Enqueue(HttpRequest request, byte[][] encodedEvents) { - var encoded = new byte[events.Count][]; - var i = 0; - foreach (var e in events) - { - var s = e.ToString(Formatting.None); - var payload = Encoding.UTF8.GetBytes(s); - - if (payload.Length > (int) _outputConfig.EventBodyLimitBytes) - { - IngestionLog.ForPayload(ClientHostIP, s).Debug("An oversized event was dropped"); - - var jo = e as JObject; - // ReSharper disable SuspiciousTypeConversion.Global - var timestamp = (string?) (dynamic?) jo?.GetValue("Timestamp") ?? DateTime.UtcNow.ToString("o"); - var level = (string?) (dynamic?) jo?.GetValue("Level") ?? "Warning"; + var apiKeyToken = request.Headers[SeqApi.ApiKeyHeaderName].FirstOrDefault(); - if (jo != null) - { - jo.Remove("Timestamp"); - jo.Remove("Level"); - } - - var startToLog = (int) Math.Min(_outputConfig.EventBodyLimitBytes / 2, 1024); - var compactPrefix = e.ToString(Formatting.None).Substring(0, startToLog); - - encoded[i] = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new - { - Timestamp = timestamp, - MessageTemplate = "Seq Forwarder received and dropped an oversized event", - Level = level, - Properties = new - { - Partial = compactPrefix, - Environment.MachineName, - _outputConfig.EventBodyLimitBytes, - PayloadBytes = payload.Length - } - })); - } - else - { - encoded[i] = payload; - } - - i++; - } + if (string.IsNullOrWhiteSpace(apiKeyToken)) + apiKeyToken = request.Query["apiKey"]; - return encoded; - } + var apiKey = string.IsNullOrWhiteSpace(apiKeyToken) + ? null + : apiKeyToken.Trim(); - IActionResult Enqueue(byte[][] encodedEvents) - { - var apiKey = GetRequestApiKeyToken(); _logBufferMap.GetLogBuffer(apiKey).Enqueue(encodedEvents); - - var response = Content(_serverResponseProxy.GetResponseText(apiKey), "application/json", Encoding); - response.StatusCode = (int)HttpStatusCode.Created; - return response; - } - - string? GetRequestApiKeyToken() - { - var apiKeyToken = Request.Headers[SeqApi.ApiKeyHeaderName].FirstOrDefault(); - - if (string.IsNullOrWhiteSpace(apiKeyToken)) - apiKeyToken = Request.Query["apiKey"]; - - var normalized = apiKeyToken?.Trim(); - if (string.IsNullOrEmpty(normalized)) - return null; - return normalized; - } + return TypedResults.Content( + _serverResponseProxy.GetResponseText(apiKey), + "application/json", + Utf8, + StatusCodes.Status201Created); - bool DefaultedBoolQuery(string queryParameterName) - { - var parameter = Request.Query[queryParameterName]; - if (parameter.Count != 1) - return false; - - var value = (string?) parameter; - - if (value == "" && ( - Request.QueryString.Value!.Contains($"&{queryParameterName}=") || - Request.QueryString.Value.Contains($"?{queryParameterName}="))) - { - return false; - } - - return "true".Equals(value, StringComparison.OrdinalIgnoreCase) || value == "" || value == queryParameterName; } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Host/Startup.cs b/src/SeqCli/Forwarder/Web/Host/Startup.cs index 804e2065..3ca712a6 100644 --- a/src/SeqCli/Forwarder/Web/Host/Startup.cs +++ b/src/SeqCli/Forwarder/Web/Host/Startup.cs @@ -8,7 +8,6 @@ class Startup { public void ConfigureServices(IServiceCollection serviceCollection) { - serviceCollection.AddMvc(); } public void Configure(IApplicationBuilder app) From a942aeb9903ccb8362ca6ea54eb62f47c3210a9b Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 1 Mar 2024 15:15:57 +1000 Subject: [PATCH 25/51] Improve registration --- src/SeqCli/Forwarder/ForwarderModule.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index a0df187d..1bccec25 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -46,7 +46,7 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType().SingleInstance(); builder.RegisterType().As(); builder.RegisterType().As(); - builder.RegisterInstance(_config.Connection); + builder.Register(c => _config.Connection); builder.RegisterInstance(new MessageTemplateTextFormatter( "[{Timestamp:o} {Level:u3}] {Message}{NewLine}" + (_config.Forwarder.Diagnostics.IngestionLogShowDetail ? "" From 7d69c41f64b10487ec5fb09fdcc79e51f516b21c Mon Sep 17 00:00:00 2001 From: KodrAus Date: Sat, 2 Mar 2024 08:57:47 +1000 Subject: [PATCH 26/51] start replacing forwarder with seqcli infra --- .../Cli/Commands/Forwarder/TruncateCommand.cs | 2 - src/SeqCli/Forwarder/ForwarderModule.cs | 6 - .../Forwarder/Multiplexing/ActiveLogBuffer.cs | 37 --- .../Multiplexing/ActiveLogBufferMap.cs | 223 ---------------- .../Multiplexing/HttpLogShipperFactory.cs | 42 --- .../Multiplexing/ILogShipperFactory.cs | 23 -- .../Multiplexing/InertLogShipperFactory.cs | 26 -- .../Multiplexing/ServerResponseProxy.cs | 51 ---- src/SeqCli/Forwarder/Schema/EventSchema.cs | 183 ------------- .../ExponentialBackoffConnectionSchedule.cs | 73 ----- .../Forwarder/Shipper/HttpLogShipper.cs | 250 ------------------ .../Forwarder/Shipper/InertLogShipper.cs | 30 --- src/SeqCli/Forwarder/Shipper/LogShipper.cs | 24 -- src/SeqCli/Forwarder/Shipper/SeqApi.cs | 20 -- .../Forwarder/Web/Api/IngestionEndpoints.cs | 197 ++++++++------ .../Forwarder/Web/Host/ServerService.cs | 9 +- .../Multiplexing/ActiveLogBufferMapTests.cs | 80 ------ .../Forwarder/Schema/EventSchemaTests.cs | 72 ----- .../Shipper/ServerResponseProxyTests.cs | 46 ---- 19 files changed, 117 insertions(+), 1277 deletions(-) delete mode 100644 src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs delete mode 100644 src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs delete mode 100644 src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs delete mode 100644 src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs delete mode 100644 src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs delete mode 100644 src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs delete mode 100644 src/SeqCli/Forwarder/Schema/EventSchema.cs delete mode 100644 src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs delete mode 100644 src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs delete mode 100644 src/SeqCli/Forwarder/Shipper/InertLogShipper.cs delete mode 100644 src/SeqCli/Forwarder/Shipper/LogShipper.cs delete mode 100644 src/SeqCli/Forwarder/Shipper/SeqApi.cs delete mode 100644 test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs delete mode 100644 test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs delete mode 100644 test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs diff --git a/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs index c73b37d6..844eb9ee 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs @@ -15,7 +15,6 @@ using System; using System.Threading.Tasks; using SeqCli.Cli.Features; -using SeqCli.Forwarder.Multiplexing; using Serilog; namespace SeqCli.Cli.Commands.Forwarder; @@ -39,7 +38,6 @@ protected override async Task Run(string[] args) if (!_confirm.TryConfirm("All data in the forwarder's log buffer will be deleted. This cannot be undone.")) return 1; - ActiveLogBufferMap.Truncate(_storagePath.BufferPath); return 0; } catch (Exception ex) diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 1bccec25..e06e09a0 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -17,7 +17,6 @@ using System.Threading; using Autofac; using SeqCli.Config; -using SeqCli.Forwarder.Multiplexing; using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; using Serilog.Formatting.Display; @@ -38,12 +37,7 @@ public ForwarderModule(string bufferPath, SeqCliConfig config) protected override void Load(ContainerBuilder builder) { builder.RegisterType().SingleInstance(); - builder.RegisterType() - .WithParameter("bufferPath", _bufferPath) - .SingleInstance(); - builder.RegisterType().As(); - builder.RegisterType().SingleInstance(); builder.RegisterType().As(); builder.RegisterType().As(); builder.Register(c => _config.Connection); diff --git a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs deleted file mode 100644 index 52fd743b..00000000 --- a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright © Datalust Pty Ltd and Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using SeqCli.Forwarder.Shipper; -using SeqCli.Forwarder.Storage; - -namespace SeqCli.Forwarder.Multiplexing; - -sealed class ActiveLogBuffer : IDisposable -{ - public LogShipper Shipper { get; } - public LogBuffer Buffer { get; } - - public ActiveLogBuffer(LogBuffer logBuffer, LogShipper logShipper) - { - Buffer = logBuffer ?? throw new ArgumentNullException(nameof(logBuffer)); - Shipper = logShipper ?? throw new ArgumentNullException(nameof(logShipper)); - } - - public void Dispose() - { - Shipper.Dispose(); - Buffer.Dispose(); - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs deleted file mode 100644 index 92f96f2b..00000000 --- a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright © Datalust Pty Ltd and Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Net; -using System.Text; -using SeqCli.Config; -using SeqCli.Encryptor; -using SeqCli.Forwarder.Storage; -using SeqCli.Forwarder.Web; -using Serilog; - -namespace SeqCli.Forwarder.Multiplexing; - -class ActiveLogBufferMap : IDisposable -{ - const string DataFileName = "data.mdb", LockFileName = "lock.mdb", ApiKeyFileName = ".apikey"; - - static readonly Encoding ApiKeyEncoding = new UTF8Encoding(false); - - readonly ulong _bufferSizeBytes; - readonly ConnectionConfig _connectionConfig; - readonly ILogShipperFactory _shipperFactory; - readonly IDataProtector _dataProtector; - readonly string _bufferPath; - readonly ILogger _log = Log.ForContext(); - - readonly object _sync = new(); - bool _loaded; - ActiveLogBuffer? _noApiKeyLogBuffer; - readonly Dictionary _buffersByApiKey = new(); - - public ActiveLogBufferMap( - string bufferPath, - SeqCliConfig config, - ILogShipperFactory logShipperFactory) - { - ArgumentNullException.ThrowIfNull(config, nameof(config)); - _bufferSizeBytes = config.Forwarder.Storage.BufferSizeBytes; - _connectionConfig = config.Connection; - _shipperFactory = logShipperFactory ?? throw new ArgumentNullException(nameof(logShipperFactory)); - _dataProtector = config.Encryption.DataProtector(); - _bufferPath = bufferPath ?? throw new ArgumentNullException(nameof(bufferPath)); - } - - // The odd three-stage initialization improves our chances of correctly tearing down the `LightningEnvironment`s within - // `LogBuffer`s in the event of a failure during start-up. See: https://github.com/CoreyKaylor/Lightning.NET/blob/master/src/LightningDB/LightningEnvironment.cs#L252 - public void Load() - { - // At startup, we look for buffers and either delete them if they're empty, or load them - // up if they're not. This garbage collection at start-up is a simplification, - // we might try cleaning up in the background if the gains are worthwhile, although more synchronization - // would be required. - - lock (_sync) - { - if (_loaded) throw new InvalidOperationException("The log buffer map is already loaded."); - - Directory.CreateDirectory(_bufferPath); - - var defaultDataFilePath = Path.Combine(_bufferPath, DataFileName); - if (File.Exists(defaultDataFilePath)) - { - _log.Information("Loading the default log buffer in {Path}", _bufferPath); - var buffer = new LogBuffer(_bufferPath, _bufferSizeBytes); - if (buffer.Peek(0).Length == 0) - { - _log.Information("The default buffer is empty and will be removed until more data is received"); - buffer.Dispose(); - File.Delete(defaultDataFilePath); - var lockFilePath = Path.Combine(_bufferPath, LockFileName); - if (File.Exists(lockFilePath)) - File.Delete(lockFilePath); - } - else - { - _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _connectionConfig.DecodeApiKey(_dataProtector))); - } - } - - foreach (var subfolder in Directory.GetDirectories(_bufferPath)) - { - var encodedApiKeyFilePath = Path.Combine(subfolder, ApiKeyFileName); - if (!File.Exists(encodedApiKeyFilePath)) - { - _log.Information("Folder {Path} does not appear to be a log buffer; skipping", subfolder); - continue; - } - - _log.Information("Loading an API-key specific buffer in {Path}", subfolder); - var apiKey = ApiKeyEncoding.GetString(_dataProtector.Decrypt(File.ReadAllBytes(encodedApiKeyFilePath))); - - var buffer = new LogBuffer(subfolder, _bufferSizeBytes); - if (buffer.Peek(0).Length == 0) - { - _log.Information("API key-specific buffer in {Path} is empty and will be removed until more data is received", subfolder); - buffer.Dispose(); - Directory.Delete(subfolder, true); - } - else - { - var activeBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, apiKey)); - _buffersByApiKey.Add(apiKey, activeBuffer); - } - } - - _loaded = true; - } - } - - public void Start() - { - lock (_sync) - { - if (!_loaded) throw new InvalidOperationException("The log buffer map is not loaded."); - - foreach (var buffer in OpenBuffers) - { - buffer.Shipper.Start(); - } - } - } - - public void Stop() - { - lock (_sync) - { - // Hard to ensure _loaded is set in all cases, better here to be forgiving and - // permit a clean shut-down. - - foreach (var buffer in OpenBuffers) - { - buffer.Shipper.Stop(); - } - } - } - - public LogBuffer GetLogBuffer(string? apiKey) - { - lock (_sync) - { - if (!_loaded) throw new RequestProcessingException("The forwarder service is starting up.", HttpStatusCode.ServiceUnavailable); - - if (apiKey == null) - { - if (_noApiKeyLogBuffer == null) - { - _log.Information("Creating a new default log buffer in {Path}", _bufferPath); - var buffer = new LogBuffer(_bufferPath, _bufferSizeBytes); - _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _connectionConfig.DecodeApiKey(_dataProtector))); - _noApiKeyLogBuffer.Shipper.Start(); - } - return _noApiKeyLogBuffer.Buffer; - } - - if (_buffersByApiKey.TryGetValue(apiKey, out var existing)) - return existing.Buffer; - - var subfolder = Path.Combine(_bufferPath, Guid.NewGuid().ToString("n")); - _log.Information("Creating a new API key-specific log buffer in {Path}", subfolder); - Directory.CreateDirectory(subfolder); - File.WriteAllBytes(Path.Combine(subfolder, ".apikey"), _dataProtector.Encrypt(ApiKeyEncoding.GetBytes(apiKey))); - var newBuffer = new LogBuffer(subfolder, _bufferSizeBytes); - var newActiveBuffer = new ActiveLogBuffer(newBuffer, _shipperFactory.Create(newBuffer, apiKey)); - _buffersByApiKey.Add(apiKey, newActiveBuffer); - newActiveBuffer.Shipper.Start(); - return newBuffer; - } - } - - public void Dispose() - { - lock (_sync) - { - foreach (var buffer in OpenBuffers) - { - buffer.Dispose(); - } - } - } - - public static void Truncate(string bufferPath) - { - DeleteIfExists(Path.Combine(bufferPath, DataFileName)); - DeleteIfExists(Path.Combine(bufferPath, LockFileName)); - foreach (var subdirectory in Directory.GetDirectories(bufferPath)) - { - if (File.Exists(Path.Combine(subdirectory, ApiKeyFileName))) - Directory.Delete(subdirectory, true); - } - } - - static void DeleteIfExists(string filePath) - { - if (File.Exists(filePath)) - File.Delete(filePath); - } - - IEnumerable OpenBuffers - { - get - { - if (_noApiKeyLogBuffer != null) - yield return _noApiKeyLogBuffer; - - foreach (var buffer in _buffersByApiKey.Values) - yield return buffer; - } - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs deleted file mode 100644 index 3101421a..00000000 --- a/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright © Datalust Pty Ltd and Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Net.Http; -using SeqCli.Config; -using SeqCli.Forwarder.Shipper; -using SeqCli.Forwarder.Storage; - -namespace SeqCli.Forwarder.Multiplexing; - -class HttpLogShipperFactory : ILogShipperFactory -{ - readonly HttpClient _outputHttpClient; - readonly ServerResponseProxy _serverResponseProxy; - readonly ConnectionConfig _outputConfig; - - public HttpLogShipperFactory(SeqCliConfig config, ServerResponseProxy serverResponseProxy, HttpClient outputHttpClient) - { - ArgumentNullException.ThrowIfNull(config, nameof(config)); - - _outputHttpClient = outputHttpClient; - _serverResponseProxy = serverResponseProxy ?? throw new ArgumentNullException(nameof(serverResponseProxy)); - _outputConfig = config.Connection; - } - - public LogShipper Create(LogBuffer logBuffer, string? apiKey) - { - return new HttpLogShipper(logBuffer, apiKey, _outputConfig, _serverResponseProxy, _outputHttpClient); - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs deleted file mode 100644 index 773f455f..00000000 --- a/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright © Datalust Pty Ltd and Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using SeqCli.Forwarder.Shipper; -using SeqCli.Forwarder.Storage; - -namespace SeqCli.Forwarder.Multiplexing; - -public interface ILogShipperFactory -{ - LogShipper Create(LogBuffer logBuffer, string? apiKey); -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs deleted file mode 100644 index b6fff878..00000000 --- a/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright © Datalust Pty Ltd and Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using SeqCli.Forwarder.Shipper; -using SeqCli.Forwarder.Storage; - -namespace SeqCli.Forwarder.Multiplexing; - -class InertLogShipperFactory : ILogShipperFactory -{ - public LogShipper Create(LogBuffer logBuffer, string? apiKey) - { - return new InertLogShipper(); - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs b/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs deleted file mode 100644 index b52dc988..00000000 --- a/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright © Datalust Pty Ltd and Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System.Collections.Generic; - -namespace SeqCli.Forwarder.Multiplexing; - -public class ServerResponseProxy -{ - const string EmptyResponse = "{}"; - - readonly object _syncRoot = new(); - readonly Dictionary _lastResponseByApiKey = new(); - string _lastNoApiKeyResponse = EmptyResponse; - - public void SuccessResponseReturned(string? apiKey, string response) - { - lock (_syncRoot) - { - if (apiKey == null) - _lastNoApiKeyResponse = response; - else - _lastResponseByApiKey[apiKey] = response; - } - } - - public string GetResponseText(string? apiKey) - { - lock (_syncRoot) - { - if (apiKey == null) - return _lastNoApiKeyResponse; - - if (_lastResponseByApiKey.TryGetValue(apiKey, out var response)) - return response; - - return EmptyResponse; - } - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Schema/EventSchema.cs b/src/SeqCli/Forwarder/Schema/EventSchema.cs deleted file mode 100644 index c13f0325..00000000 --- a/src/SeqCli/Forwarder/Schema/EventSchema.cs +++ /dev/null @@ -1,183 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Globalization; -using System.Linq; -using Newtonsoft.Json.Linq; -using SeqCli.Forwarder.Util; -using Serilog.Parsing; - -namespace SeqCli.Forwarder.Schema; - -static class EventSchema -{ - static readonly MessageTemplateParser MessageTemplateParser = new(); - - static readonly HashSet ClefReifiedProperties = ["@t", "@m", "@mt", "@l", "@x", "@i", "@r"]; - - public static bool FromClefFormat(in int lineNumber, JObject compactFormat, [MaybeNullWhen(false)] out JObject rawFormat, [MaybeNullWhen(true)] out string error) - { - var result = new JObject(); - - var rawTimestamp = compactFormat["@t"]; - if (rawTimestamp == null) - { - error = $"The event on line {lineNumber} does not carry an `@t` timestamp property."; - rawFormat = default; - return false; - } - - if (rawTimestamp.Type != JTokenType.String) - { - error = $"The event on line {lineNumber} has an invalid `@t` timestamp property; the value must be a JSON string."; - rawFormat = default; - return false; - } - - if (!DateTimeOffset.TryParse(rawTimestamp.Value(), out _)) - { - error = $"The timestamp value `{rawTimestamp}` on line {lineNumber} could not be parsed."; - rawFormat = default; - return false; - } - - result.Add("Timestamp", rawTimestamp); - - var properties = new JObject(); - foreach (var property in compactFormat.Properties()) - { - if (property.Name.StartsWith("@@")) - properties.Add(property.Name.Substring(1), property.Value); - else if (!ClefReifiedProperties.Contains(property.Name)) - properties.Add(property.Name, property.Value); - } - - var x = compactFormat["@x"]; - if (x != null) - { - if (x.Type != JTokenType.String) - { - error = $"The event on line {lineNumber} has a non-string `@x` exception property."; - rawFormat = default; - return false; - } - - result.Add("Exception", x); - } - - var l = compactFormat["@l"]; - if (l != null) - { - if (l.Type != JTokenType.String) - { - error = $"The event on line {lineNumber} has a non-string `@l` level property."; - rawFormat = default; - return false; - } - - result.Add("Level", l); - } - - string? message = null; - var m = compactFormat["@m"]; - if (m != null) - { - if (m.Type != JTokenType.String) - { - error = $"The event on line {lineNumber} has a non-string `@m` message property."; - rawFormat = default; - return false; - } - - message = m.Value(); - } - - string? messageTemplate = null; - var mt = compactFormat["@mt"]; - if (mt != null) - { - if (mt.Type != JTokenType.String) - { - error = $"The event on line {lineNumber} has a non-string `@mt` message template property."; - rawFormat = default; - return false; - } - - messageTemplate = mt.Value(); - } - - if (message != null) - { - result.Add("RenderedMessage", message); - } - else if (messageTemplate != null && compactFormat["@r"] is JArray renderingsArray) - { - var template = MessageTemplateParser.Parse(messageTemplate); - var withFormat = template.Tokens.OfType().Where(pt => pt.Format != null); - - // ReSharper disable once PossibleMultipleEnumeration - if (withFormat.Count() == renderingsArray.Count) - { - // ReSharper disable once PossibleMultipleEnumeration - var renderingsByProperty = withFormat - .Zip(renderingsArray, (p, j) => new { p.PropertyName, Format = p.Format!, Rendering = j.Value() }) - .GroupBy(p => p.PropertyName) - .ToDictionary(g => g.Key, g => g.ToDictionaryDistinct(p => p.Format, p => p.Rendering)); - - var renderings = new JObject(); - result.Add("Renderings", renderings); - - foreach (var (property, propertyRenderings) in renderingsByProperty) - { - var byFormat = new JArray(); - renderings.Add(property, byFormat); - - foreach (var (format, rendering) in propertyRenderings) - { - var element = new JObject {{"Format", format}, {"Rendering", rendering}}; - byFormat.Add(element); - } - } - } - } - - messageTemplate ??= message ?? "No template provided"; - result.Add("MessageTemplate", messageTemplate); - - var eventTypeToken = compactFormat["@i"]; - if (eventTypeToken != null) - { - if (eventTypeToken.Type == JTokenType.Integer) - { - result.Add("EventType", uint.Parse(eventTypeToken.Value()!)); - } - else if (eventTypeToken.Type == JTokenType.String) - { - if (uint.TryParse(eventTypeToken.Value(), NumberStyles.HexNumber, - CultureInfo.InvariantCulture, out var eventType)) - { - result.Add("EventType", eventType); - } - else - { - // Seq would calculate a hash value from the string, here. Forwarder will ignore that - // case and preserve the value in an `@i` property for now. - result.Add("@i", eventTypeToken); - } - } - else - { - error = $"The `@i` event type value on line {lineNumber} is not in a string or numeric format."; - rawFormat = default; - return false; - } - } - - if (properties.Count != 0) - result.Add("Properties", properties); - - rawFormat = result; - error = null; - return true; - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs b/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs deleted file mode 100644 index 9439f4ba..00000000 --- a/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright © Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; - -namespace SeqCli.Forwarder.Shipper; - -class ExponentialBackoffConnectionSchedule -{ - static readonly TimeSpan MinimumBackoffPeriod = TimeSpan.FromSeconds(5); - static readonly TimeSpan MaximumBackoffInterval = TimeSpan.FromMinutes(10); - - readonly TimeSpan _period; - - int _failuresSinceSuccessfulConnection; - - public ExponentialBackoffConnectionSchedule(TimeSpan period) - { - if (period < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(period), "The connection retry period must be a positive timespan."); - - _period = period; - } - - public void MarkSuccess() - { - _failuresSinceSuccessfulConnection = 0; - } - - public void MarkFailure() - { - ++_failuresSinceSuccessfulConnection; - } - - public bool LastConnectionFailed => _failuresSinceSuccessfulConnection != 0; - - public TimeSpan NextInterval - { - get - { - // Available, and first failure, just try the batch interval - if (_failuresSinceSuccessfulConnection <= 1) return _period; - - // Second failure, start ramping up the interval - first 2x, then 4x, ... - var backoffFactor = Math.Pow(2, (_failuresSinceSuccessfulConnection - 1)); - - // If the period is ridiculously short, give it a boost so we get some - // visible backoff. - var backoffPeriod = Math.Max(_period.Ticks, MinimumBackoffPeriod.Ticks); - - // The "ideal" interval - var backedOff = (long)(backoffPeriod * backoffFactor); - - // Capped to the maximum interval - var cappedBackoff = Math.Min(MaximumBackoffInterval.Ticks, backedOff); - - // Unless that's shorter than the base interval, in which case we'll just apply the period - var actual = Math.Max(_period.Ticks, cappedBackoff); - - return TimeSpan.FromTicks(actual); - } - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs b/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs deleted file mode 100644 index f39159b1..00000000 --- a/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright © Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.IO; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using SeqCli.Config; -using SeqCli.Forwarder.Multiplexing; -using SeqCli.Forwarder.Storage; -using SeqCli.Forwarder.Util; -using Serilog; - -namespace SeqCli.Forwarder.Shipper; - -sealed class HttpLogShipper : LogShipper -{ - const string BulkUploadResource = "api/events/raw"; - - readonly string? _apiKey; - readonly LogBuffer _logBuffer; - readonly ConnectionConfig _outputConfig; - readonly HttpClient _httpClient; - readonly ExponentialBackoffConnectionSchedule _connectionSchedule; - readonly ServerResponseProxy _serverResponseProxy; - DateTime _nextRequiredLevelCheck; - - readonly object _stateLock = new(); - readonly Timer _timer; - bool _started; - - volatile bool _unloading; - - static readonly TimeSpan QuietWaitPeriod = TimeSpan.FromSeconds(2), MaximumConnectionInterval = TimeSpan.FromMinutes(2); - - public HttpLogShipper(LogBuffer logBuffer, string? apiKey, ConnectionConfig outputConfig, ServerResponseProxy serverResponseProxy, HttpClient outputHttpClient) - { - _apiKey = apiKey; - _httpClient = outputHttpClient ?? throw new ArgumentNullException(nameof(outputHttpClient)); - _logBuffer = logBuffer ?? throw new ArgumentNullException(nameof(logBuffer)); - _outputConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); - _serverResponseProxy = serverResponseProxy ?? throw new ArgumentNullException(nameof(serverResponseProxy)); - _connectionSchedule = new ExponentialBackoffConnectionSchedule(QuietWaitPeriod); - _timer = new Timer(_ => OnTick()); - } - - public override void Start() - { - lock (_stateLock) - { - if (_started) - throw new InvalidOperationException("The shipper has already started."); - - if (_unloading) - throw new InvalidOperationException("The shipper is unloading."); - - Log.Information("Log shipper started, events will be dispatched to {ServerUrl}", _outputConfig.ServerUrl); - - _nextRequiredLevelCheck = DateTime.UtcNow.Add(MaximumConnectionInterval); - _started = true; - SetTimer(); - } - } - - public override void Stop() - { - lock (_stateLock) - { - if (_unloading) - return; - - _unloading = true; - - if (!_started) - return; - } - - var wh = new ManualResetEvent(false); - if (_timer.Dispose(wh)) - wh.WaitOne(); - } - - public override void Dispose() - { - Stop(); - } - - void SetTimer() - { - _timer.Change(_connectionSchedule.NextInterval, Timeout.InfiniteTimeSpan); - } - - void OnTick() - { - OnTickAsync().Wait(); - } - - async Task OnTickAsync() - { - try - { - var sendingSingles = 0; - do - { - var available = _logBuffer.Peek((int)_outputConfig.PayloadLimitBytes); - if (available.Length == 0) - { - if (DateTime.UtcNow < _nextRequiredLevelCheck || _connectionSchedule.LastConnectionFailed) - { - // For whatever reason, there's nothing waiting to send. This means we should try connecting again at the - // regular interval, so mark the attempt as successful. - _connectionSchedule.MarkSuccess(); - break; - } - } - - MakePayload(available, sendingSingles > 0, out Stream payload, out ulong lastIncluded); - - var content = new StreamContent(new UnclosableStreamWrapper(payload)); - content.Headers.ContentType = new MediaTypeHeaderValue("application/json") - { - CharSet = Encoding.UTF8.WebName - }; - - if (_apiKey != null) - { - content.Headers.Add(SeqApi.ApiKeyHeaderName, _apiKey); - } - - var result = await _httpClient.PostAsync(BulkUploadResource, content); - if (result.IsSuccessStatusCode) - { - _connectionSchedule.MarkSuccess(); - _logBuffer.Dequeue(lastIncluded); - if (sendingSingles > 0) - sendingSingles--; - - _serverResponseProxy.SuccessResponseReturned(_apiKey, await result.Content.ReadAsStringAsync()); - _nextRequiredLevelCheck = DateTime.UtcNow.Add(MaximumConnectionInterval); - } - else if (result.StatusCode == HttpStatusCode.BadRequest || - result.StatusCode == HttpStatusCode.RequestEntityTooLarge) - { - // The connection attempt was successful - the payload we sent was the problem. - _connectionSchedule.MarkSuccess(); - - if (sendingSingles != 0) - { - payload.Position = 0; - var payloadText = await new StreamReader(payload, Encoding.UTF8).ReadToEndAsync(); - Log.Error("HTTP shipping failed with {StatusCode}: {Result}; payload was {InvalidPayload}", result.StatusCode, await result.Content.ReadAsStringAsync(), payloadText); - _logBuffer.Dequeue(lastIncluded); - sendingSingles = 0; - } - else - { - // Unscientific (should "binary search" in batches) but sending the next - // hundred events singly should flush out the problematic one. - sendingSingles = 100; - } - } - else - { - _connectionSchedule.MarkFailure(); - Log.Error("Received failed HTTP shipping result {StatusCode}: {Result}", result.StatusCode, await result.Content.ReadAsStringAsync()); - break; - } - } - while (true); - } - catch (HttpRequestException hex) - { - Log.Warning(hex, "HTTP request failed when sending a batch from the log shipper"); - _connectionSchedule.MarkFailure(); - } - catch (Exception ex) - { - Log.Error(ex, "Exception while sending a batch from the log shipper"); - _connectionSchedule.MarkFailure(); - } - finally - { - lock (_stateLock) - { - if (!_unloading) - SetTimer(); - } - } - } - - void MakePayload(LogBufferEntry[] entries, bool oneOnly, out Stream utf8Payload, out ulong lastIncluded) - { - if (entries == null) throw new ArgumentNullException(nameof(entries)); - lastIncluded = 0; - - var raw = new MemoryStream(); - var content = new StreamWriter(raw, Encoding.UTF8); - content.Write("{\"Events\":["); - content.Flush(); - var contentRemainingBytes = (int) _outputConfig.PayloadLimitBytes - 13; // Includes closing delimiters - - var delimStart = ""; - foreach (var logBufferEntry in entries) - { - if ((ulong)logBufferEntry.Value.Length > _outputConfig.EventBodyLimitBytes) - { - Log.Information("Oversize event will be skipped, {Payload}", Encoding.UTF8.GetString(logBufferEntry.Value)); - lastIncluded = logBufferEntry.Key; - continue; - } - - // lastIncluded indicates we've added at least one event - if (lastIncluded != 0 && contentRemainingBytes - (delimStart.Length + logBufferEntry.Value.Length) < 0) - break; - - content.Write(delimStart); - content.Flush(); - contentRemainingBytes -= delimStart.Length; - - raw.Write(logBufferEntry.Value, 0, logBufferEntry.Value.Length); - contentRemainingBytes -= logBufferEntry.Value.Length; - - lastIncluded = logBufferEntry.Key; - - delimStart = ","; - if (oneOnly) - break; - } - - content.Write("]}"); - content.Flush(); - raw.Position = 0; - utf8Payload = raw; - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs b/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs deleted file mode 100644 index 1ae106e3..00000000 --- a/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright © Datalust Pty Ltd and Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -namespace SeqCli.Forwarder.Shipper; - -class InertLogShipper : LogShipper -{ - public override void Start() - { - } - - public override void Stop() - { - } - - public override void Dispose() - { - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Shipper/LogShipper.cs b/src/SeqCli/Forwarder/Shipper/LogShipper.cs deleted file mode 100644 index 83e8beb3..00000000 --- a/src/SeqCli/Forwarder/Shipper/LogShipper.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright © Datalust Pty Ltd and Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; - -namespace SeqCli.Forwarder.Shipper; - -public abstract class LogShipper : IDisposable -{ - public abstract void Start(); - public abstract void Stop(); - public abstract void Dispose(); -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Shipper/SeqApi.cs b/src/SeqCli/Forwarder/Shipper/SeqApi.cs deleted file mode 100644 index 5e7c45e2..00000000 --- a/src/SeqCli/Forwarder/Shipper/SeqApi.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright © Datalust Pty Ltd and Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -namespace SeqCli.Forwarder.Shipper; - -static class SeqApi -{ - public const string ApiKeyHeaderName = "X-Seq-ApiKey"; -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index 3a463c9d..24592474 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -13,24 +13,22 @@ // limitations under the License. using System; +using System.Buffers; using System.Collections.Generic; -using System.IO; -using System.Linq; using System.Net; using System.Text; +using System.Text.Json; using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Http.Features; using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.Net.Http.Headers; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using SeqCli.Config; using SeqCli.Forwarder.Diagnostics; -using SeqCli.Forwarder.Multiplexing; -using SeqCli.Forwarder.Schema; -using SeqCli.Forwarder.Shipper; +using JsonException = System.Text.Json.JsonException; +using JsonSerializer = Newtonsoft.Json.JsonSerializer; namespace SeqCli.Forwarder.Web.Api; @@ -38,21 +36,15 @@ class IngestionEndpoints : IMapEndpoints { static readonly Encoding Utf8 = new UTF8Encoding(false); - readonly ActiveLogBufferMap _logBufferMap; readonly ConnectionConfig _connectionConfig; - readonly ServerResponseProxy _serverResponseProxy; readonly JsonSerializer _rawSerializer = JsonSerializer.Create( new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); public IngestionEndpoints( - ActiveLogBufferMap logBufferMap, - ServerResponseProxy serverResponseProxy, ConnectionConfig connectionConfig) { - _logBufferMap = logBufferMap; _connectionConfig = connectionConfig; - _serverResponseProxy = serverResponseProxy; } public void Map(WebApplication app) @@ -77,7 +69,7 @@ public void Map(WebApplication app) })); } - byte[][] EncodeRawEvents(ICollection events, IPAddress remoteIpAddress) + IEnumerable EncodeRawEvents(ICollection events, IPAddress remoteIpAddress) { var encoded = new byte[events.Count][]; var i = 0; @@ -149,98 +141,141 @@ static bool DefaultedBoolQuery(HttpRequest request, string queryParameterName) IResult IngestRawFormat(HttpContext context) { - // The compact format ingestion path works with async IO. - context.Features.Get()!.AllowSynchronousIO = true; - - JObject posted; - try - { - posted = _rawSerializer.Deserialize(new JsonTextReader(new StreamReader(context.Request.Body))) ?? - throw new RequestProcessingException("Request body payload is JSON `null`."); - } - catch (Exception ex) - { - IngestionLog.ForClient(context.Connection.RemoteIpAddress!).Debug(ex,"Rejecting payload due to invalid JSON, request body could not be parsed"); - throw new RequestProcessingException("Invalid raw event JSON, body could not be parsed."); - } - - if (!(posted.TryGetValue("events", StringComparison.Ordinal, out var eventsToken) || - posted.TryGetValue("Events", StringComparison.Ordinal, out eventsToken))) - { - IngestionLog.ForClient(context.Connection.RemoteIpAddress!).Debug("Rejecting payload due to invalid JSON structure"); - throw new RequestProcessingException("Invalid raw event JSON, body must contain an 'Events' array."); - } - - if (!(eventsToken is JArray events)) - { - IngestionLog.ForClient(context.Connection.RemoteIpAddress!).Debug("Rejecting payload due to invalid Events property structure"); - throw new RequestProcessingException("Invalid raw event JSON, the 'Events' property must be an array."); - } - - var encoded = EncodeRawEvents(events, context.Connection.RemoteIpAddress!); - return Enqueue(context.Request, encoded); + // Convert legacy format to CLEF + throw new NotImplementedException(); } async Task IngestCompactFormat(HttpContext context) { - var rawFormat = new List(); - var reader = new StreamReader(context.Request.Body); - - var line = await reader.ReadLineAsync(); - var lineNumber = 1; + var payload = ArrayPool.Shared.Rent(1024 * 1024 * 10); + var writeHead = 0; + var readHead = 0; + var discarding = false; - while (line != null) + var done = false; + while (!done) { - if (!string.IsNullOrWhiteSpace(line)) + // Fill our buffer + while (!done) { - JObject item; - try + var remaining = payload.Length - writeHead; + if (remaining == 0) { - item = _rawSerializer.Deserialize(new JsonTextReader(new StringReader(line))) ?? - throw new RequestProcessingException("Request body payload is JSON `null`."); + break; } - catch (Exception ex) + + var read = await context.Request.Body.ReadAsync(payload.AsMemory(writeHead, remaining), context.RequestAborted); + if (read == 0) { - IngestionLog.ForPayload(context.Connection.RemoteIpAddress!, line).Debug(ex, "Rejecting CLEF payload due to invalid JSON, item could not be parsed"); - throw new RequestProcessingException($"Invalid raw event JSON, item on line {lineNumber} could not be parsed."); + done = true; } - if (!EventSchema.FromClefFormat(lineNumber, item, out var evt, out var err)) + writeHead += read; + } + + // Process events + var batchStart = readHead; + var batchEnd = readHead; + while (batchEnd < writeHead) + { + var eventStart = batchEnd; + var nlIndex = payload.AsSpan()[eventStart..].IndexOf((byte)'\n'); + + if (nlIndex == -1) { - IngestionLog.ForPayload(context.Connection.RemoteIpAddress!, line).Debug("Rejecting CLEF payload due to invalid event JSON structure: {NormalizationError}", err); - throw new RequestProcessingException(err); + break; } - rawFormat.Add(evt); - } + var eventEnd = eventStart + nlIndex + 1; - line = await reader.ReadLineAsync(); - ++lineNumber; - } + if (discarding) + { + batchStart = eventEnd; + batchEnd = eventEnd; + readHead = batchEnd; - return Enqueue( - context.Request, - EncodeRawEvents(rawFormat, context.Connection.RemoteIpAddress!)); - } - - ContentHttpResult Enqueue(HttpRequest request, byte[][] encodedEvents) - { - var apiKeyToken = request.Headers[SeqApi.ApiKeyHeaderName].FirstOrDefault(); + discarding = false; + } + else + { + batchEnd = eventEnd; + readHead = batchEnd; - if (string.IsNullOrWhiteSpace(apiKeyToken)) - apiKeyToken = request.Query["apiKey"]; + if (!ValidateClef(payload.AsSpan()[eventStart..batchEnd])) + { + Enqueue(payload.AsSpan()[batchStart..eventStart]); + batchStart = batchEnd; + } + } + } + + if (batchStart != batchEnd) + { + Enqueue(payload.AsSpan()[batchStart..batchEnd]); + } + else if (batchStart == 0) + { + readHead = payload.Length; + discarding = true; + } - var apiKey = string.IsNullOrWhiteSpace(apiKeyToken) - ? null - : apiKeyToken.Trim(); + // Copy any unprocessed data into our buffer and continue + if (!done) + { + var retain = payload.Length - readHead; + payload.AsSpan()[retain..].CopyTo(payload.AsSpan()[..retain]); + readHead = retain; + writeHead = retain; + } + } - _logBufferMap.GetLogBuffer(apiKey).Enqueue(encodedEvents); + ArrayPool.Shared.Return(payload); return TypedResults.Content( - _serverResponseProxy.GetResponseText(apiKey), + null, "application/json", Utf8, StatusCodes.Status201Created); - + } + + bool ValidateClef(Span evt) + { + var reader = new Utf8JsonReader(evt); + + try + { + reader.Read(); + if (reader.TokenType != JsonTokenType.StartObject) + { + return false; + } + + while (reader.Read()) + { + if (reader.CurrentDepth == 1) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + var name = reader.GetString(); + + if (name != null & name!.StartsWith("@")) + { + // Validate @ property + } + } + } + } + } + catch (JsonException) + { + return false; + } + + return true; + } + + void Enqueue(Span payload) + { + // Do the thing } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Host/ServerService.cs b/src/SeqCli/Forwarder/Web/Host/ServerService.cs index 9118eb24..40934120 100644 --- a/src/SeqCli/Forwarder/Web/Host/ServerService.cs +++ b/src/SeqCli/Forwarder/Web/Host/ServerService.cs @@ -15,20 +15,17 @@ using System; using Microsoft.Extensions.Hosting; using SeqCli.Forwarder.Diagnostics; -using SeqCli.Forwarder.Multiplexing; using Serilog; namespace SeqCli.Forwarder.Web.Host; class ServerService { - readonly ActiveLogBufferMap _logBufferMap; readonly IHost _host; readonly string _listenUri; - public ServerService(ActiveLogBufferMap logBufferMap, IHost host, string listenUri) + public ServerService(IHost host, string listenUri) { - _logBufferMap = logBufferMap; _host = host; _listenUri = listenUri; } @@ -43,9 +40,6 @@ public void Start() Log.Information("Seq Forwarder listening on {ListenUri}", _listenUri); IngestionLog.Log.Debug("Seq Forwarder is accepting events"); - - _logBufferMap.Load(); - _logBufferMap.Start(); } catch (Exception ex) { @@ -59,7 +53,6 @@ public void Stop() Log.Debug("Seq Forwarder stopping"); _host.StopAsync().Wait(); - _logBufferMap.Stop(); Log.Information("Seq Forwarder stopped cleanly"); } diff --git a/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs deleted file mode 100644 index a4ca2484..00000000 --- a/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs +++ /dev/null @@ -1,80 +0,0 @@ -using System.IO; -using System.Linq; -using SeqCli.Config; -using SeqCli.Forwarder.Multiplexing; -using SeqCli.Tests.Support; -using Xunit; - -namespace SeqCli.Tests.Forwarder.Multiplexing; - -public class ActiveLogBufferMapTests -{ - [Fact] - public void AnEmptyMapCreatesNoFiles() - { - using var tmp = new TempFolder("Buffer"); - using var map = CreateActiveLogBufferMap(tmp); - Assert.Empty(Directory.GetFileSystemEntries(tmp.Path)); - } - - [Fact] - public void TheDefaultBufferWritesDataInTheBufferRoot() - { - using var tmp = new TempFolder("Buffer"); - using var map = CreateActiveLogBufferMap(tmp); - var entry = map.GetLogBuffer(null); - Assert.NotNull(entry); - Assert.True(File.Exists(Path.Combine(tmp.Path, "data.mdb"))); - Assert.Empty(Directory.GetDirectories(tmp.Path)); - Assert.Same(entry, map.GetLogBuffer(null)); - } - - [Fact] - public void ApiKeySpecificBuffersWriteDataToSubfolders() - { - using var tmp = new TempFolder("Buffer"); - using var map = CreateActiveLogBufferMap(tmp); - string key1 = Some.ApiKey(), key2 = Some.ApiKey(); - var entry1 = map.GetLogBuffer(key1); - var entry2 = map.GetLogBuffer(key2); - - Assert.NotNull(entry1); - Assert.NotNull(entry2); - Assert.Same(entry1, map.GetLogBuffer(key1)); - Assert.NotSame(entry1, entry2); - var subdirs = Directory.GetDirectories(tmp.Path); - Assert.Equal(2, subdirs.Length); - Assert.True(File.Exists(Path.Combine(subdirs[0], "data.mdb"))); - Assert.True(File.Exists(Path.Combine(subdirs[0], ".apikey"))); - } - - [Fact] - public void EntriesSurviveReloads() - { - var apiKey = Some.ApiKey(); - var value = Some.Bytes(100); - - using var tmp = new TempFolder("Buffer"); - using (var map = CreateActiveLogBufferMap(tmp)) - { - map.GetLogBuffer(null).Enqueue([value]); - map.GetLogBuffer(apiKey).Enqueue([value]); - } - - using (var map = CreateActiveLogBufferMap(tmp)) - { - var first = map.GetLogBuffer(null).Peek(0).Single(); - var second = map.GetLogBuffer(apiKey).Peek(0).Single(); - Assert.Equal(value, first.Value); - Assert.Equal(value, second.Value); - } - } - - static ActiveLogBufferMap CreateActiveLogBufferMap(TempFolder tmp) - { - var config = new SeqCliConfig(); - var map = new ActiveLogBufferMap(tmp.Path, config, new InertLogShipperFactory()); - map.Load(); - return map; - } -} \ No newline at end of file diff --git a/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs b/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs deleted file mode 100644 index 8d1abcba..00000000 --- a/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs +++ /dev/null @@ -1,72 +0,0 @@ -using System.IO; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using SeqCli.Forwarder.Schema; -using Xunit; - -namespace SeqCli.Tests.Forwarder.Schema; - -public class EventSchemaTests -{ - static readonly JsonSerializer RawSerializer = JsonSerializer.Create( - new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); - - [Fact] - public void ClefNormalizationAcceptsDuplicateRenderings() - { - var payload = "{\"@t\": \"2015-05-09T12:09:08.12345Z\"," + - " \"@mt\": \"{A:000} and {A:000}\"," + - " \"@r\": [\"424\",\"424\"]}"; - - AssertCanNormalizeClef(payload); - } - - [Fact] - public void ClefNormalizationPropagatesRenderings() - { - const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12,\"@r\":[\"012\"]}"; - var evt = AssertCanNormalizeClef(payload); - Assert.Single(evt.Renderings); - } - - [Fact] - public void ClefNormalizationIgnoresMissingRenderings() - { - const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12}"; - AssertCanNormalizeClef(payload); - } - - [Fact] - public void ClefNormalizationFixesTooFewRenderings1() - { - const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12,\"@r\":[]}"; - var evt = AssertCanNormalizeClef(payload); - Assert.Null(evt.Renderings); - } - - [Fact] - public void ClefNormalizationFixesTooFewRenderings2() - { - const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000} {Q:x}!\",\"P\":12,\"@r\":[\"012\"]}"; - var evt = AssertCanNormalizeClef(payload); - Assert.Null(evt.Renderings); - } - - [Fact] - public void ClefNormalizationIgnoresTooManyRenderings() - { - const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12,\"@r\":[\"012\",\"013\"]}"; - var evt = AssertCanNormalizeClef(payload); - Assert.Null(evt.Renderings); - } - - static dynamic AssertCanNormalizeClef(string payload) - { - var jo = RawSerializer.Deserialize(new JsonTextReader(new StringReader(payload)))!; - - var valid = EventSchema.FromClefFormat(1, jo, out var rawFormat, out var error); - Assert.True(valid, error); - Assert.NotNull(rawFormat); - return rawFormat!; - } -} \ No newline at end of file diff --git a/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs b/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs deleted file mode 100644 index 107a6973..00000000 --- a/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs +++ /dev/null @@ -1,46 +0,0 @@ -using SeqCli.Forwarder.Multiplexing; -using SeqCli.Tests.Support; -using Xunit; - -namespace SeqCli.Tests.Forwarder.Shipper; - -public class ServerResponseProxyTests -{ - [Fact] - public void WhenNoResponseRecordedEmptyIsReturned() - { - var proxy = new ServerResponseProxy(); - var response = proxy.GetResponseText(Some.ApiKey()); - Assert.Equal("{}", response); - } - - [Fact] - public void WhenApiKeysDontMatchEmptyResponseReturned() - { - var proxy = new ServerResponseProxy(); - proxy.SuccessResponseReturned(Some.ApiKey(), "this is never used"); - var response = proxy.GetResponseText(Some.ApiKey()); - Assert.Equal("{}", response); - } - - [Fact] - public void WhenApiKeysMatchTheResponseIsReturned() - { - var proxy = new ServerResponseProxy(); - var apiKey = Some.ApiKey(); - var responseText = "some response"; - proxy.SuccessResponseReturned(apiKey, responseText); - var response = proxy.GetResponseText(apiKey); - Assert.Equal(responseText, response); - } - - [Fact] - public void NullApiKeysAreConsideredMatching() - { - var proxy = new ServerResponseProxy(); - var responseText = "some response"; - proxy.SuccessResponseReturned(null, responseText); - var response = proxy.GetResponseText(null); - Assert.Equal(responseText, response); - } -} \ No newline at end of file From e6875c49b9ca57b0635166c916aab6eb94c8f23d Mon Sep 17 00:00:00 2001 From: KodrAus Date: Mon, 4 Mar 2024 08:04:59 +1000 Subject: [PATCH 27/51] fill in clef ingest --- src/SeqCli/Forwarder/ForwarderModule.cs | 4 +- src/SeqCli/Forwarder/Storage/LogBuffer.cs | 302 +++--------------- .../Forwarder/Storage/LogBufferEntry.cs | 23 +- src/SeqCli/Forwarder/Storage/LogBufferMap.cs | 17 + .../Forwarder/Web/Api/IngestionEndpoints.cs | 45 ++- 5 files changed, 100 insertions(+), 291 deletions(-) create mode 100644 src/SeqCli/Forwarder/Storage/LogBufferMap.cs diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index e06e09a0..9f75f2dc 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -17,6 +17,7 @@ using System.Threading; using Autofac; using SeqCli.Config; +using SeqCli.Forwarder.Storage; using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; using Serilog.Formatting.Display; @@ -37,10 +38,10 @@ public ForwarderModule(string bufferPath, SeqCliConfig config) protected override void Load(ContainerBuilder builder) { builder.RegisterType().SingleInstance(); + builder.RegisterType().SingleInstance(); builder.RegisterType().As(); builder.RegisterType().As(); - builder.Register(c => _config.Connection); builder.RegisterInstance(new MessageTemplateTextFormatter( "[{Timestamp:o} {Level:u3}] {Message}{NewLine}" + (_config.Forwarder.Diagnostics.IngestionLogShowDetail ? "" @@ -73,7 +74,6 @@ protected override void Load(ContainerBuilder builder) } return new HttpClient { BaseAddress = new Uri(baseUri) }; - }).SingleInstance(); builder.RegisterInstance(_config); diff --git a/src/SeqCli/Forwarder/Storage/LogBuffer.cs b/src/SeqCli/Forwarder/Storage/LogBuffer.cs index f58fe8db..b53236cd 100644 --- a/src/SeqCli/Forwarder/Storage/LogBuffer.cs +++ b/src/SeqCli/Forwarder/Storage/LogBuffer.cs @@ -1,279 +1,57 @@ -// Copyright © Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - using System; -using System.Collections.Generic; -using Serilog; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; namespace SeqCli.Forwarder.Storage; -public class LogBuffer : IDisposable +record LogBuffer { - readonly ulong _bufferSizeBytes; - // readonly LightningEnvironment _env; - readonly object _sync = new(); - bool _isDisposed; - ulong _nextId = 0, _entries = 0, _writtenSinceRotateCheck; - - public LogBuffer(string bufferPath, ulong bufferSizeBytes) - { - _bufferSizeBytes = bufferSizeBytes; - if (bufferPath == null) throw new ArgumentNullException(nameof(bufferPath)); - - // _env = new LightningEnvironment(bufferPath) - // { - // // Sparse; we'd hope fragmentation never gets this bad... - // MapSize = (long) bufferSizeBytes*10 - // }; - // - // _env.Open(); - // - // using (var tx = _env.BeginTransaction()) - // using (var db = tx.OpenDatabase()) - // { - // using (var cur = tx.CreateCursor(db)) - // { - // if (!cur.MoveToLast()) - // { - // _nextId = 1; - // } - // else - // { - // var current = cur.GetCurrent(); - // _nextId = ByteKeyToULongKey(current.Key) + 1; - // _entries = (ulong) tx.GetEntriesCount(db); - // } - // } - // } - - Log.Information("Log buffer open on {BufferPath}; {Entries} entries, next key will be {NextId}", bufferPath, _entries, _nextId); - } - - public void Dispose() + public LogBuffer(Func write, CancellationToken cancellationToken) { - lock (_sync) + var channel = Channel.CreateBounded(new BoundedChannelOptions(5) + { + SingleReader = false, + SingleWriter = true, + FullMode = BoundedChannelFullMode.Wait, + }); + + _shutdownTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + _writer = channel.Writer; + _worker = Task.Run(async () => { - if (!_isDisposed) + await foreach (var entry in channel.Reader.ReadAllAsync(_shutdownTokenSource.Token)) { - _isDisposed = true; - // _env.Dispose(); + try + { + await write(_shutdownTokenSource.Token); + entry.Completion.SetResult(); + } + catch (Exception e) + { + entry.Completion.TrySetException(e); + } } - } - } - - public void Enqueue(byte[][] values) - { - if (values == null) throw new ArgumentNullException(nameof(values)); - - lock (_sync) - { - RequireNotDisposed(); - - // var totalPayloadWritten = 0UL; - // - // using (var tx = _env.BeginTransaction()) - // using (var db = tx.OpenDatabase()) - // { - // foreach (var v in values) - // { - // if (v == null) throw new ArgumentException("Value array may not contain null."); - // - // tx.Put(db, ULongKeyToByteKey(_nextId++), v); - // totalPayloadWritten += (ulong) v.Length; - // } - // - // tx.Commit(); - // _entries += (ulong) values.Length; - // _writtenSinceRotateCheck += totalPayloadWritten; - // } - - RotateIfRequired(); - } + }, cancellationToken: _shutdownTokenSource.Token); } - - void RotateIfRequired() + + readonly ChannelWriter _writer; + readonly Task _worker; + readonly CancellationTokenSource _shutdownTokenSource; + + public async Task WriteAsync(byte[] storage, Range range, CancellationToken cancellationToken) { - if (_writtenSinceRotateCheck < _bufferSizeBytes/10) - return; + var tcs = new TaskCompletionSource(); + var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownTokenSource.Token); - _writtenSinceRotateCheck = 0; - // - // using (var tx = _env.BeginTransaction()) - // using (var db = tx.OpenDatabase()) - // { - // int err; - // if (0 != (err = Lmdb.mdb_env_info(_env.Handle(), out var estat))) - // throw new Exception(Lmdb.mdb_strerror(err)); - // - // MDBStat stat; - // if (0 != (err = Lmdb.mdb_stat(tx.Handle(), db.Handle(), out stat))) - // throw new Exception(Lmdb.mdb_strerror(err)); - // - // // http://www.openldap.org/lists/openldap-technical/201303/msg00145.html - // // 1) MDB_stat gives you the page size. - // // 2) MDB_envinfo tells the mapsize and the last_pgno.If you divide mapsize - // // by pagesize you'll get max pgno. The MAP_FULL error is returned when last_pgno reaches max pgno. - // - // var targetPages = _bufferSizeBytes/stat.ms_psize; - // if ((ulong) estat.me_last_pgno < targetPages && (double) (ulong) estat.me_last_pgno/targetPages < 0.75) - // return; - // - // var count = tx.GetEntriesCount(db); - // if (count == 0) - // { - // Log.Warning("Attempting to rotate buffer but no events are present"); - // return; - // } - // - // var toPurge = Math.Max(count / 4, 1); - // Log.Warning("Buffer is full; dropping {ToPurge} events to make room for new ones", - // toPurge); - // - // using (var cur = tx.CreateCursor(db)) - // { - // cur.MoveToFirst(); - // - // for (var i = 0; i < toPurge; ++i) - // { - // cur.Delete(); - // cur.MoveNext(); - // } - // } - // - // tx.Commit(); - // } + await _writer.WriteAsync(new LogBufferEntry(storage, range, tcs), cts.Token); + await tcs.Task; } - public LogBufferEntry[] Peek(int maxValueBytesHint) + public async Task StopAsync() { - lock (_sync) - { - RequireNotDisposed(); - - var entries = new List(); - // - // using (var tx = _env.BeginTransaction(TransactionBeginFlags.ReadOnly)) - // using (var db = tx.OpenDatabase()) - // { - // using (var cur = tx.CreateCursor(db)) - // { - // if (cur.MoveToFirst()) - // { - // var entriesBytes = 0; - // - // do - // { - // var current = cur.GetCurrent(); - // var entry = new LogBufferEntry - // { - // Key = ByteKeyToULongKey(current.Key), - // Value = current.Value - // }; - // - // entriesBytes += entry.Value.Length; - // if (entries.Count != 0 && entriesBytes > maxValueBytesHint) - // break; - // - // entries.Add(entry); - // - // } while (cur.MoveNext()); - // } - // } - // } - - return entries.ToArray(); - } - } - - public void Dequeue(ulong toKey) - { - lock (_sync) - { - RequireNotDisposed(); - - // ulong deleted = 0; - // - // using (var tx = _env.BeginTransaction()) - // using (var db = tx.OpenDatabase()) - // { - // using (var cur = tx.CreateCursor(db)) - // { - // if (cur.MoveToFirst()) - // { - // do - // { - // var current = cur.GetCurrent(); - // if (ByteKeyToULongKey(current.Key) > toKey) - // break; - // - // cur.Delete(); - // deleted++; - // } while (cur.MoveNext()); - // } - // } - // - // tx.Commit(); - // _entries -= deleted; - // } - } - } - - void RequireNotDisposed() - { - if (_isDisposed) - throw new ObjectDisposedException(typeof(LogBuffer).FullName); - } - - static ulong ByteKeyToULongKey(byte[] key) - { - var copy = new byte[key.Length]; - for (var i = 0; i < key.Length; ++i) - copy[copy.Length - (i + 1)] = key[i]; - - return BitConverter.ToUInt64(copy, 0); - } - - static byte[] ULongKeyToByteKey(ulong key) - { - var k = BitConverter.GetBytes(key); - Array.Reverse(k); - return k; - } - - public void Enumerate(Action action) - { - if (action == null) throw new ArgumentNullException(nameof(action)); - - lock (_sync) - { - RequireNotDisposed(); - - // using (var tx = _env.BeginTransaction(TransactionBeginFlags.ReadOnly)) - // using (var db = tx.OpenDatabase()) - // { - // using (var cur = tx.CreateCursor(db)) - // { - // if (cur.MoveToFirst()) - // { - // do - // { - // var current = cur.GetCurrent(); - // action(ByteKeyToULongKey(current.Key), current.Value); - // } while (cur.MoveNext()); - // } - // } - // } - } + _writer.Complete(); + await _worker; + await _shutdownTokenSource.CancelAsync(); } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs b/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs index 649be980..4477b926 100644 --- a/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs +++ b/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs @@ -1,23 +1,6 @@ -// Copyright © Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// ReSharper disable InconsistentNaming +using System; +using System.Threading.Tasks; namespace SeqCli.Forwarder.Storage; -public struct LogBufferEntry -{ - public ulong Key; - public byte[] Value; -} \ No newline at end of file +public readonly record struct LogBufferEntry(byte[] Storage, Range Range, TaskCompletionSource Completion); diff --git a/src/SeqCli/Forwarder/Storage/LogBufferMap.cs b/src/SeqCli/Forwarder/Storage/LogBufferMap.cs new file mode 100644 index 00000000..b5b246fa --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/LogBufferMap.cs @@ -0,0 +1,17 @@ +using System; +using System.Threading.Tasks; + +namespace SeqCli.Forwarder.Storage; + +class LogBufferMap +{ + public LogBufferMap() + { + + } + + public LogBuffer Get(string? apiKey) + { + return new LogBuffer(async (c) => await Task.Delay(TimeSpan.FromSeconds(1), c), default); + } +} diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index 24592474..48fc8fa2 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -15,9 +15,11 @@ using System; using System.Buffers; using System.Collections.Generic; +using System.Linq; using System.Net; using System.Text; using System.Text.Json; +using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; @@ -27,6 +29,7 @@ using Newtonsoft.Json.Linq; using SeqCli.Config; using SeqCli.Forwarder.Diagnostics; +using SeqCli.Forwarder.Storage; using JsonException = System.Text.Json.JsonException; using JsonSerializer = Newtonsoft.Json.JsonSerializer; @@ -37,14 +40,17 @@ class IngestionEndpoints : IMapEndpoints static readonly Encoding Utf8 = new UTF8Encoding(false); readonly ConnectionConfig _connectionConfig; + readonly LogBufferMap _logBuffers; readonly JsonSerializer _rawSerializer = JsonSerializer.Create( new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); public IngestionEndpoints( - ConnectionConfig connectionConfig) + SeqCliConfig config, + LogBufferMap logBuffers) { - _connectionConfig = connectionConfig; + _connectionConfig = config.Connection; + _logBuffers = logBuffers; } public void Map(WebApplication app) @@ -138,6 +144,17 @@ static bool DefaultedBoolQuery(HttpRequest request, string queryParameterName) return "true".Equals(value, StringComparison.OrdinalIgnoreCase) || value == "" || value == queryParameterName; } + + static string? ApiKey(HttpRequest request) + { + var apiKeyHeader = request.Headers["X-SeqApiKey"]; + + if (apiKeyHeader.Count > 0) return apiKeyHeader.Last(); + if (request.Query.TryGetValue("apiKey", out var apiKey)) return apiKey.Last(); + + return null; + } + IResult IngestRawFormat(HttpContext context) { @@ -147,6 +164,11 @@ IResult IngestRawFormat(HttpContext context) async Task IngestCompactFormat(HttpContext context) { + var cts = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted); + cts.CancelAfter(TimeSpan.FromSeconds(5)); + + var log = _logBuffers.Get(ApiKey(context.Request)); + var payload = ArrayPool.Shared.Rent(1024 * 1024 * 10); var writeHead = 0; var readHead = 0; @@ -203,7 +225,7 @@ async Task IngestCompactFormat(HttpContext context) if (!ValidateClef(payload.AsSpan()[eventStart..batchEnd])) { - Enqueue(payload.AsSpan()[batchStart..eventStart]); + await Write(log, ArrayPool.Shared, payload, batchStart..eventStart, cts.Token); batchStart = batchEnd; } } @@ -211,7 +233,7 @@ async Task IngestCompactFormat(HttpContext context) if (batchStart != batchEnd) { - Enqueue(payload.AsSpan()[batchStart..batchEnd]); + await Write(log, ArrayPool.Shared, payload, batchStart..batchEnd, cts.Token); } else if (batchStart == 0) { @@ -229,8 +251,9 @@ async Task IngestCompactFormat(HttpContext context) } } + // Exception cases are handled by `Write` ArrayPool.Shared.Return(payload); - + return TypedResults.Content( null, "application/json", @@ -274,8 +297,16 @@ bool ValidateClef(Span evt) return true; } - void Enqueue(Span payload) + async Task Write(LogBuffer log, ArrayPool pool, byte[] storage, Range range, CancellationToken cancellationToken) { - // Do the thing + try + { + await log.WriteAsync(storage, range, cancellationToken); + } + catch + { + pool.Return(storage); + throw; + } } } \ No newline at end of file From dbc154d8b7769a20e2eb0524527b794e94881074 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 8 May 2024 15:52:39 +1000 Subject: [PATCH 28/51] Tidy up some shut-down logic --- .DS_Store | Bin 0 -> 6148 bytes .../Cli/Commands/Forwarder/InstallCommand.cs | 8 +- .../Cli/Commands/Forwarder/RestartCommand.cs | 2 +- .../Cli/Commands/Forwarder/RunCommand.cs | 64 +++++--- .../Cli/Commands/Forwarder/StartCommand.cs | 2 +- .../Cli/Commands/Forwarder/StatusCommand.cs | 6 +- .../Cli/Commands/Forwarder/StopCommand.cs | 2 +- .../Commands/Forwarder/UninstallCommand.cs | 4 +- .../Cli/Features/ServiceCredentialsFeature.cs | 2 +- src/SeqCli/Cli/Features/StoragePathFeature.cs | 4 +- .../Forwarder/Diagnostics/InMemorySink.cs | 2 +- .../Forwarder/Diagnostics/IngestionLog.cs | 2 +- .../SeqCliForwarderWindowsService.cs | 4 +- src/SeqCli/Forwarder/Storage/LogBuffer.cs | 4 +- src/SeqCli/Forwarder/Storage/LogBufferMap.cs | 7 + .../Forwarder/Util/AccountRightsHelper.cs | 2 +- .../Forwarder/Util/EnumerableExtensions.cs | 19 --- .../Forwarder/Util/ExecutionEnvironment.cs | 2 +- .../Forwarder/Util/ServiceConfiguration.cs | 2 +- src/SeqCli/Forwarder/Util/WindowsProcess.cs | 2 +- .../Forwarder/Web/Api/ApiRootEndpoints.cs | 28 +--- src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs | 2 +- .../Forwarder/Web/Api/IngestionEndpoints.cs | 80 ++-------- .../Web/Api/IngestionLogEndpoints.cs | 51 ++++++ .../Forwarder/Web/Host/ServerService.cs | 20 ++- src/SeqCli/Forwarder/Web/Host/Startup.cs | 38 ----- .../Forwarder/Storage/LogBufferTests.cs | 149 ------------------ test/SeqCli.Tests/SeqCli.Tests.csproj | 1 - test/SeqCli.Tests/Support/TempFolder.cs | 2 +- 29 files changed, 153 insertions(+), 358 deletions(-) create mode 100644 .DS_Store delete mode 100644 src/SeqCli/Forwarder/Util/EnumerableExtensions.cs create mode 100644 src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs delete mode 100644 src/SeqCli/Forwarder/Web/Host/Startup.cs delete mode 100644 test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..1a8b21bcb239f7d0ec1a07dcfc24e16d43c2a3a8 GIT binary patch literal 6148 zcmeHK-AcnS6i&A3I)<rAAeI4GAHmGT z{yN~d*IC9=_7D``zdw$$+_pb?t=`z$Zdy&NZQXfKvhZ?0pJyXKy+P|z$~dTWKe&pf z#iX-)CX?Kcl4+_6qHqc+x7Se;%3>tvNtmfxPY0}))tYn;7K@|4GwgMRWA~k9S1g7- zyDJ7KgXOYi?HwMUUQC|jmqfm4MmZ3!WXE6y@1U3ky?SX9%j6L}Rd$s{NDL4I!~ij{ z!3>x~!D??X2efozfEZZM0PYV08lr2k)Tp)&=7p`UpzfkFnyBet{28e-G2I|`E;Q4 Run(string[] unrecognized) try { + // ISSUE: we can't really rely on the default `SeqCliConfig` path being readable when running as a service. config = SeqCliConfig.Read(); // _storagePath.ConfigFilePath); } catch (Exception ex) @@ -107,7 +107,7 @@ protected override async Task Run(string[] unrecognized) { options.AddServerHeader = false; options.AllowSynchronousIO = true; - }).ConfigureKestrel((context, options) => + }).ConfigureKestrel((_, options) => { var apiListenUri = new Uri(listenUri); @@ -125,8 +125,8 @@ protected override async Task Run(string[] unrecognized) options.Listen(ipAddress, apiListenUri.Port, listenOptions => { #if WINDOWS - listenOptions.UseHttps(StoreName.My, apiListenUri.Host, - location: StoreLocation.LocalMachine, allowInvalid: true); + listenOptions.UseHttps(StoreName.My, apiListenUri.Host, + location: StoreLocation.LocalMachine, allowInvalid: true); #else listenOptions.UseHttps(); #endif @@ -138,31 +138,52 @@ protected override async Task Run(string[] unrecognized) } }); - builder - .Host.UseSerilog() - .UseServiceProviderFactory(new AutofacServiceProviderFactory()) - .ConfigureContainer(builder => + builder.Services.AddSerilog(); + + builder.Host.UseServiceProviderFactory(new AutofacServiceProviderFactory()) + .ConfigureContainer(containerBuilder => { - builder.RegisterBuildCallback(ls => container = ls); - builder.RegisterModule(new ForwarderModule(_storagePath.BufferPath, config)); + containerBuilder.RegisterBuildCallback(ls => container = ls); + containerBuilder.RegisterModule(new ForwarderModule(_storagePath.BufferPath, config)); }); - using var host = builder.Build(); + await using var app = builder.Build(); if (container == null) throw new Exception("Host did not build container."); + app.Use(async (context, next) => + { + try + { + await next(); + } + // ISSUE: this exception type isn't currently used. + catch (RequestProcessingException rex) + { + if (context.Response.HasStarted) + throw; + + context.Response.StatusCode = (int)rex.StatusCode; + context.Response.ContentType = "text/plain; charset=UTF-8"; + await context.Response.Body.WriteAsync(Encoding.UTF8.GetBytes(rex.Message)); + await context.Response.CompleteAsync(); + } + }); + foreach (var mapper in container.Resolve>()) { - mapper.Map(host); + mapper.MapEndpoints(app); } var service = container.Resolve( - new TypedParameter(typeof(IHost), host), + new TypedParameter(typeof(IHost), app), new NamedParameter("listenUri", listenUri)); var exit = ExecutionEnvironment.SupportsStandardIO - ? RunStandardIO(service, Console.Out) + ? await RunStandardIOAsync(service, Console.Out) : RunService(service); + + Log.Information("Exiting with status code {StatusCode}", exit); return exit; } @@ -178,6 +199,7 @@ protected override async Task Run(string[] unrecognized) } [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] + // ReSharper disable once UnusedParameter.Local static int RunService(ServerService service) { #if WINDOWS @@ -190,7 +212,7 @@ static int RunService(ServerService service) #endif } - static int RunStandardIO(ServerService service, TextWriter cout) + static async Task RunStandardIOAsync(ServerService service, TextWriter cout) { service.Start(); @@ -210,7 +232,7 @@ static int RunStandardIO(ServerService service, TextWriter cout) Console.Read(); } - service.Stop(); + await service.StopAsync(); return 0; } @@ -219,9 +241,9 @@ static void WriteBanner() { Write("─", ConsoleColor.DarkGray, 47); Console.WriteLine(); - Write(" Seq Forwarder", ConsoleColor.White); + Write(" SeqCli Forwarder", ConsoleColor.White); Write(" ──", ConsoleColor.DarkGray); - Write(" © 2024 Datalust Pty Ltd", ConsoleColor.Gray); + Write(" © Datalust Pty Ltd and Contributors", ConsoleColor.Gray); Console.WriteLine(); Write("─", ConsoleColor.DarkGray, 47); Console.WriteLine(); @@ -244,7 +266,7 @@ static Logger CreateLogger( var loggerConfiguration = new LoggerConfiguration() .Enrich.FromLogContext() .Enrich.WithProperty("MachineName", Environment.MachineName) - .Enrich.WithProperty("Application", "Seq Forwarder") + .Enrich.WithProperty("Application", "SeqCli Forwarder") .MinimumLevel.Is(internalLoggingLevel) .MinimumLevel.Override("Microsoft", LogEventLevel.Warning) .WriteTo.File( diff --git a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs index dcbefb38..98ee92f8 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs @@ -22,7 +22,7 @@ using SeqCli.Cli; using SeqCli.Forwarder.ServiceProcess; -namespace Seq.Forwarder.Cli.Commands +namespace SeqCli.Forwarder.Cli.Commands { [Command("forwarder", "start", "Start the forwarder Windows service")] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] diff --git a/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs index 3eb6b0d3..3d0073b1 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs @@ -22,7 +22,7 @@ using SeqCli.Cli; using SeqCli.Forwarder.ServiceProcess; -namespace Seq.Forwarder.Cli.Commands +namespace SeqCli.Forwarder.Cli.Commands { [Command("forwarder", "status", "Show the status of the forwarder Windows service")] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] @@ -33,11 +33,11 @@ protected override Task Run() try { var controller = new ServiceController(SeqCliForwarderWindowsService.WindowsServiceName); - Console.WriteLine("The Seq Forwarder service is installed and {0}.", controller.Status.ToString().ToLowerInvariant()); + Console.WriteLine($"The {SeqCliForwarderWindowsService.WindowsServiceName} service is installed and {controller.Status.ToString().ToLowerInvariant()}."); } catch (InvalidOperationException) { - Console.WriteLine("The Seq Forwarder service is not installed."); + Console.WriteLine($"The {SeqCliForwarderWindowsService.WindowsServiceName} service is not installed."); } catch (Exception ex) { diff --git a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs index 59761a19..88d7db6b 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs @@ -22,7 +22,7 @@ using SeqCli.Cli; using SeqCli.Forwarder.ServiceProcess; -namespace Seq.Forwarder.Cli.Commands +namespace SeqCli.Forwarder.Cli.Commands { [Command("forwarder", "stop", "Stop the forwarder Windows service")] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] diff --git a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs index 7ca95726..224cc94c 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs @@ -17,12 +17,12 @@ using System; using System.IO; using System.Threading.Tasks; -using Seq.Forwarder.Util; +using SeqCli.Forwarder.Util; using SeqCli.Cli; using SeqCli.Forwarder.ServiceProcess; using SeqCli.Forwarder.Util; -namespace Seq.Forwarder.Cli.Commands +namespace SeqCli.Forwarder.Cli.Commands { [Command("forwarder", "uninstall", "Uninstall the forwarder Windows service")] class UninstallCommand : Command diff --git a/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs b/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs index 7e7fcd1a..60c2d250 100644 --- a/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs +++ b/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs @@ -16,7 +16,7 @@ using SeqCli.Cli; -namespace Seq.Forwarder.Cli.Features +namespace SeqCli.Forwarder.Cli.Features { class ServiceCredentialsFeature : CommandFeature { diff --git a/src/SeqCli/Cli/Features/StoragePathFeature.cs b/src/SeqCli/Cli/Features/StoragePathFeature.cs index 4283ddea..809a62b6 100644 --- a/src/SeqCli/Cli/Features/StoragePathFeature.cs +++ b/src/SeqCli/Cli/Features/StoragePathFeature.cs @@ -41,14 +41,14 @@ static string GetDefaultStorageRoot() // Specific to and writable by the current user. Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), #endif - "Seq", + "SeqCli", "Forwarder")); } static string? TryQueryInstalledStorageRoot() { #if WINDOWS - if (Seq.Forwarder.Util.ServiceConfiguration.GetServiceStoragePath( + if (SeqCli.Forwarder.Util.ServiceConfiguration.GetServiceStoragePath( SeqCliForwarderWindowsService.WindowsServiceName, out var storage)) return storage; #endif diff --git a/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs b/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs index b5eea21f..6375dcf9 100644 --- a/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs +++ b/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs @@ -37,7 +37,7 @@ public IEnumerable Read() public void Emit(LogEvent logEvent) { - if (logEvent == null) throw new ArgumentNullException(nameof(logEvent)); + ArgumentNullException.ThrowIfNull(logEvent); _queue.Enqueue(logEvent); while (_queue.Count > _queueLength) diff --git a/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs b/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs index 5e54a5bf..e3fbadf2 100644 --- a/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs +++ b/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs @@ -42,7 +42,7 @@ public static IEnumerable Read() return Sink.Read(); } - public static ILogger ForClient(IPAddress clientHostIP) + public static ILogger ForClient(IPAddress? clientHostIP) { return Log.ForContext("ClientHostIP", clientHostIP); } diff --git a/src/SeqCli/Forwarder/ServiceProcess/SeqCliForwarderWindowsService.cs b/src/SeqCli/Forwarder/ServiceProcess/SeqCliForwarderWindowsService.cs index b0885128..3dfbc691 100644 --- a/src/SeqCli/Forwarder/ServiceProcess/SeqCliForwarderWindowsService.cs +++ b/src/SeqCli/Forwarder/ServiceProcess/SeqCliForwarderWindowsService.cs @@ -26,7 +26,7 @@ class SeqCliForwarderWindowsService : ServiceBase { readonly ServerService _serverService; - public static string WindowsServiceName { get; } = "Seq Forwarder"; + public static string WindowsServiceName { get; } = "SeqCli Forwarder"; public SeqCliForwarderWindowsService(ServerService serverService) { @@ -46,7 +46,7 @@ protected override void OnStart(string[] args) protected override void OnStop() { - _serverService.Stop(); + _serverService.StopAsync().Wait(); } } } diff --git a/src/SeqCli/Forwarder/Storage/LogBuffer.cs b/src/SeqCli/Forwarder/Storage/LogBuffer.cs index b53236cd..25bde0d9 100644 --- a/src/SeqCli/Forwarder/Storage/LogBuffer.cs +++ b/src/SeqCli/Forwarder/Storage/LogBuffer.cs @@ -5,7 +5,7 @@ namespace SeqCli.Forwarder.Storage; -record LogBuffer +class LogBuffer { public LogBuffer(Func write, CancellationToken cancellationToken) { @@ -42,7 +42,7 @@ public LogBuffer(Func write, CancellationToken cancella public async Task WriteAsync(byte[] storage, Range range, CancellationToken cancellationToken) { var tcs = new TaskCompletionSource(); - var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownTokenSource.Token); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownTokenSource.Token); await _writer.WriteAsync(new LogBufferEntry(storage, range, tcs), cts.Token); await tcs.Task; diff --git a/src/SeqCli/Forwarder/Storage/LogBufferMap.cs b/src/SeqCli/Forwarder/Storage/LogBufferMap.cs index b5b246fa..d9601814 100644 --- a/src/SeqCli/Forwarder/Storage/LogBufferMap.cs +++ b/src/SeqCli/Forwarder/Storage/LogBufferMap.cs @@ -1,5 +1,6 @@ using System; using System.Threading.Tasks; +using Serilog; namespace SeqCli.Forwarder.Storage; @@ -14,4 +15,10 @@ public LogBuffer Get(string? apiKey) { return new LogBuffer(async (c) => await Task.Delay(TimeSpan.FromSeconds(1), c), default); } + + public Task StopAsync() + { + Log.Information("Flushing log buffers"); + return Task.CompletedTask; + } } diff --git a/src/SeqCli/Forwarder/Util/AccountRightsHelper.cs b/src/SeqCli/Forwarder/Util/AccountRightsHelper.cs index 6b73caf8..9074c532 100644 --- a/src/SeqCli/Forwarder/Util/AccountRightsHelper.cs +++ b/src/SeqCli/Forwarder/Util/AccountRightsHelper.cs @@ -11,7 +11,7 @@ // ReSharper disable FieldCanBeMadeReadOnly.Local -namespace Seq.Forwarder.Util +namespace SeqCli.Forwarder.Util { public static class AccountRightsHelper { diff --git a/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs b/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs deleted file mode 100644 index 6f78a435..00000000 --- a/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace SeqCli.Forwarder.Util; - -static class EnumerableExtensions -{ - public static Dictionary ToDictionaryDistinct( - this IEnumerable enumerable, Func keySelector, Func valueSelector) - where TKey: notnull - { - var result = new Dictionary(); - foreach (var e in enumerable) - { - result[keySelector(e)] = valueSelector(e); - } - return result; - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs b/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs index 40b47071..4025c3eb 100644 --- a/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs +++ b/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs @@ -1,5 +1,5 @@ #if WINDOWS -using Seq.Forwarder.Util; +using SeqCli.Forwarder.Util; #endif namespace SeqCli.Forwarder.Util; diff --git a/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs b/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs index f1739326..598a2e3c 100644 --- a/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs +++ b/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs @@ -22,7 +22,7 @@ using System.Text; using SeqCli.Forwarder.Util; -namespace Seq.Forwarder.Util +namespace SeqCli.Forwarder.Util { [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] public static class ServiceConfiguration diff --git a/src/SeqCli/Forwarder/Util/WindowsProcess.cs b/src/SeqCli/Forwarder/Util/WindowsProcess.cs index 98a20930..8e4d96a2 100644 --- a/src/SeqCli/Forwarder/Util/WindowsProcess.cs +++ b/src/SeqCli/Forwarder/Util/WindowsProcess.cs @@ -8,7 +8,7 @@ // ReSharper disable once InconsistentNaming -namespace Seq.Forwarder.Util +namespace SeqCli.Forwarder.Util { static class WindowsProcess { diff --git a/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs index 90015f1b..822ecb23 100644 --- a/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs @@ -12,41 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System.IO; using System.Text; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; -using SeqCli.Forwarder.Diagnostics; -using Serilog.Formatting.Display; namespace SeqCli.Forwarder.Web.Api; class ApiRootEndpoints : IMapEndpoints { - readonly MessageTemplateTextFormatter _formatter; - readonly Encoding Utf8 = new UTF8Encoding(false); + readonly Encoding _utf8 = new UTF8Encoding(false); - public ApiRootEndpoints(MessageTemplateTextFormatter formatter) + public void MapEndpoints(WebApplication app) { - _formatter = formatter; - } - - public void Map(WebApplication app) - { - app.MapGet("/", () => - { - var events = IngestionLog.Read(); - using var log = new StringWriter(); - foreach (var logEvent in events) - { - _formatter.Format(logEvent, log); - } - - return Results.Content(log.ToString(), "text/plain", Utf8); - }); - app.MapGet("/api", - () => Results.Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Utf8)); - + () => Results.Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", _utf8)); } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs index b5c53b1b..2a0812a5 100644 --- a/src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs @@ -4,5 +4,5 @@ namespace SeqCli.Forwarder.Web.Api; interface IMapEndpoints { - void Map(WebApplication app); + void MapEndpoints(WebApplication app); } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index 48fc8fa2..e511336e 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -53,11 +53,8 @@ public IngestionEndpoints( _logBuffers = logBuffers; } - public void Map(WebApplication app) + public void MapEndpoints(WebApplication app) { - app.MapGet("/api", - () => Results.Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Utf8)); - app.MapPost("api/events/raw", new Func>(async (context) => { var clef = DefaultedBoolQuery(context.Request, "clef"); @@ -66,67 +63,17 @@ public void Map(WebApplication app) return await IngestCompactFormat(context); var contentType = (string?) context.Request.Headers[HeaderNames.ContentType]; - var clefMediaType = "application/vnd.serilog.clef"; + const string clefMediaType = "application/vnd.serilog.clef"; if (contentType != null && contentType.StartsWith(clefMediaType)) return await IngestCompactFormat(context); - return IngestRawFormat(context); + IngestionLog.ForClient(context.Connection.RemoteIpAddress) + .Error("Client supplied a legacy raw-format (non-CLEF) payload"); + return Results.BadRequest("Only newline-delimited JSON (CLEF) payloads are supported."); })); } - IEnumerable EncodeRawEvents(ICollection events, IPAddress remoteIpAddress) - { - var encoded = new byte[events.Count][]; - var i = 0; - foreach (var e in events) - { - var s = e.ToString(Formatting.None); - var payload = Utf8.GetBytes(s); - - if (payload.Length > (int) _connectionConfig.EventBodyLimitBytes) - { - IngestionLog.ForPayload(remoteIpAddress, s).Debug("An oversized event was dropped"); - - var jo = e as JObject; - // ReSharper disable SuspiciousTypeConversion.Global - var timestamp = (string?) (dynamic?) jo?.GetValue("Timestamp") ?? DateTime.UtcNow.ToString("o"); - var level = (string?) (dynamic?) jo?.GetValue("Level") ?? "Warning"; - - if (jo != null) - { - jo.Remove("Timestamp"); - jo.Remove("Level"); - } - - var startToLog = (int) Math.Min(_connectionConfig.EventBodyLimitBytes / 2, 1024); - var compactPrefix = e.ToString(Formatting.None).Substring(0, startToLog); - - encoded[i] = Utf8.GetBytes(JsonConvert.SerializeObject(new - { - Timestamp = timestamp, - MessageTemplate = "Seq Forwarder received and dropped an oversized event", - Level = level, - Properties = new - { - Partial = compactPrefix, - Environment.MachineName, - _connectionConfig.EventBodyLimitBytes, - PayloadBytes = payload.Length - } - })); - } - else - { - encoded[i] = payload; - } - - i++; - } - - return encoded; - } - static bool DefaultedBoolQuery(HttpRequest request, string queryParameterName) { var parameter = request.Query[queryParameterName]; @@ -150,16 +97,7 @@ static bool DefaultedBoolQuery(HttpRequest request, string queryParameterName) var apiKeyHeader = request.Headers["X-SeqApiKey"]; if (apiKeyHeader.Count > 0) return apiKeyHeader.Last(); - if (request.Query.TryGetValue("apiKey", out var apiKey)) return apiKey.Last(); - - return null; - } - - - IResult IngestRawFormat(HttpContext context) - { - // Convert legacy format to CLEF - throw new NotImplementedException(); + return request.Query.TryGetValue("apiKey", out var apiKey) ? apiKey.Last() : null; } async Task IngestCompactFormat(HttpContext context) @@ -261,7 +199,7 @@ async Task IngestCompactFormat(HttpContext context) StatusCodes.Status201Created); } - bool ValidateClef(Span evt) + static bool ValidateClef(Span evt) { var reader = new Utf8JsonReader(evt); @@ -281,7 +219,7 @@ bool ValidateClef(Span evt) { var name = reader.GetString(); - if (name != null & name!.StartsWith("@")) + if (name != null & name!.StartsWith($"@")) { // Validate @ property } @@ -297,7 +235,7 @@ bool ValidateClef(Span evt) return true; } - async Task Write(LogBuffer log, ArrayPool pool, byte[] storage, Range range, CancellationToken cancellationToken) + static async Task Write(LogBuffer log, ArrayPool pool, byte[] storage, Range range, CancellationToken cancellationToken) { try { diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs new file mode 100644 index 00000000..2cbb3f8f --- /dev/null +++ b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs @@ -0,0 +1,51 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.IO; +using System.Text; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using SeqCli.Forwarder.Diagnostics; +using Serilog.Formatting.Display; + +namespace SeqCli.Forwarder.Web.Api; + +class IngestionLogEndpoints : IMapEndpoints +{ + readonly MessageTemplateTextFormatter _formatter; + readonly Encoding _utf8 = new UTF8Encoding(false); + + public IngestionLogEndpoints(MessageTemplateTextFormatter formatter) + { + _formatter = formatter; + } + + public void MapEndpoints(WebApplication app) + { + // ISSUE: this route should probably only be mapped when some kind of --unsafe-debug flag + // is set. + + app.MapGet("/", () => + { + var events = IngestionLog.Read(); + using var log = new StringWriter(); + foreach (var logEvent in events) + { + _formatter.Format(logEvent, log); + } + + return Results.Content(log.ToString(), "text/plain", _utf8); + }); + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Host/ServerService.cs b/src/SeqCli/Forwarder/Web/Host/ServerService.cs index 40934120..ee4b473b 100644 --- a/src/SeqCli/Forwarder/Web/Host/ServerService.cs +++ b/src/SeqCli/Forwarder/Web/Host/ServerService.cs @@ -13,8 +13,10 @@ // limitations under the License. using System; +using System.Threading.Tasks; using Microsoft.Extensions.Hosting; using SeqCli.Forwarder.Diagnostics; +using SeqCli.Forwarder.Storage; using Serilog; namespace SeqCli.Forwarder.Web.Host; @@ -22,11 +24,13 @@ namespace SeqCli.Forwarder.Web.Host; class ServerService { readonly IHost _host; + readonly LogBufferMap _logBufferMap; readonly string _listenUri; - public ServerService(IHost host, string listenUri) + public ServerService(IHost host, LogBufferMap logBufferMap, string listenUri) { _host = host; + _logBufferMap = logBufferMap; _listenUri = listenUri; } @@ -38,8 +42,8 @@ public void Start() _host.Start(); - Log.Information("Seq Forwarder listening on {ListenUri}", _listenUri); - IngestionLog.Log.Debug("Seq Forwarder is accepting events"); + Log.Information("SeqCli Forwarder listening on {ListenUri}", _listenUri); + IngestionLog.Log.Debug("SeqCli Forwarder is accepting events"); } catch (Exception ex) { @@ -48,12 +52,14 @@ public void Start() } } - public void Stop() + public async Task StopAsync() { - Log.Debug("Seq Forwarder stopping"); + Log.Debug("Stopping HTTP server..."); - _host.StopAsync().Wait(); + await _host.StopAsync(); - Log.Information("Seq Forwarder stopped cleanly"); + Log.Information("HTTP server stopped; flushing buffers..."); + + await _logBufferMap.StopAsync(); } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Host/Startup.cs b/src/SeqCli/Forwarder/Web/Host/Startup.cs deleted file mode 100644 index 3ca712a6..00000000 --- a/src/SeqCli/Forwarder/Web/Host/Startup.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System.Text; -using Microsoft.AspNetCore.Builder; -using Microsoft.Extensions.DependencyInjection; - -namespace SeqCli.Forwarder.Web.Host; - -class Startup -{ - public void ConfigureServices(IServiceCollection serviceCollection) - { - } - - public void Configure(IApplicationBuilder app) - { - app.Use(async (context, next) => - { - try - { - await next(); - } - catch (RequestProcessingException rex) - { - if (context.Response.HasStarted) - throw; - - context.Response.StatusCode = (int)rex.StatusCode; - context.Response.ContentType = "text/plain; charset=UTF-8"; - await context.Response.Body.WriteAsync(Encoding.UTF8.GetBytes(rex.Message)); - await context.Response.CompleteAsync(); - } - }); - app.UseRouting(); - app.UseEndpoints(endpoints => - { - endpoints.MapControllers(); - }); - } -} \ No newline at end of file diff --git a/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs b/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs deleted file mode 100644 index cee028fc..00000000 --- a/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs +++ /dev/null @@ -1,149 +0,0 @@ -using System.Collections.Generic; -using SeqCli.Forwarder.Storage; -using SeqCli.Tests.Support; -using Xunit; - -namespace SeqCli.Tests.Forwarder.Storage; - -public class LogBufferTests -{ - const ulong DefaultBufferSize = 10 * 1024 * 1024; - - [Fact] - public void ANewLogBufferIsEmpty() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - var contents = buffer.Peek((int)DefaultBufferSize); - Assert.Empty(contents); - } - - [Fact] - public void PeekingDoesNotChangeState() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - buffer.Enqueue([Some.Bytes(140)]); - - var contents = buffer.Peek((int)DefaultBufferSize); - Assert.Single(contents); - - var remainder = buffer.Peek((int)DefaultBufferSize); - Assert.Single(remainder); - } - - [Fact] - public void EnqueuedEntriesAreDequeuedFifo() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2]); - buffer.Enqueue([a3]); - - var contents = buffer.Peek((int)DefaultBufferSize); - - Assert.Equal(3, contents.Length); - Assert.Equal(a1, contents[0].Value); - Assert.Equal(a2, contents[1].Value); - Assert.Equal(a3, contents[2].Value); - } - - [Fact] - public void EntriesOverLimitArePurgedFifo() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), 4096); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); - - var contents = buffer.Peek((int)DefaultBufferSize); - - Assert.Equal(2, contents.Length); - Assert.Equal(a2, contents[0].Value); - Assert.Equal(a3, contents[1].Value); - } - - [Fact] - public void SizeHintLimitsDequeuedEventCount() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); - - var contents = buffer.Peek(300); - - Assert.Equal(2, contents.Length); - Assert.Equal(a1, contents[0].Value); - Assert.Equal(a2, contents[1].Value); - } - - [Fact] - public void AtLeastOneEventIsAlwaysDequeued() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); - - var contents = buffer.Peek(30); - - Assert.Single(contents); - Assert.Equal(a1, contents[0].Value); - } - - [Fact] - public void GivingTheLastSeenEventKeyRemovesPrecedingEvents() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); - - var contents = buffer.Peek(420); - Assert.Equal(3, contents.Length); - - buffer.Dequeue(contents[2].Key); - - var remaining = buffer.Peek(420); - Assert.Empty(remaining); - } - - [Fact] - public void GivingTheLastSeeEventKeyLeavesSuccessiveEvents() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); - - var contents = buffer.Peek(30); - Assert.Single(contents); - - buffer.Enqueue([Some.Bytes(140)]); - - buffer.Dequeue(contents[0].Key); - - var remaining = buffer.Peek(420); - Assert.Equal(3, remaining.Length); - } - - [Fact] - public void EnumerationIsInOrder() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); - - var contents = new List(); - buffer.Enumerate((k, v) => - { - contents.Add(v); - }); - - Assert.Equal(3, contents.Count); - Assert.Equal(new[] { a1, a2, a3 }, contents); - } -} \ No newline at end of file diff --git a/test/SeqCli.Tests/SeqCli.Tests.csproj b/test/SeqCli.Tests/SeqCli.Tests.csproj index e93f0a96..9ebc2e63 100644 --- a/test/SeqCli.Tests/SeqCli.Tests.csproj +++ b/test/SeqCli.Tests/SeqCli.Tests.csproj @@ -15,7 +15,6 @@ - diff --git a/test/SeqCli.Tests/Support/TempFolder.cs b/test/SeqCli.Tests/Support/TempFolder.cs index 968fd857..8a5e85a0 100644 --- a/test/SeqCli.Tests/Support/TempFolder.cs +++ b/test/SeqCli.Tests/Support/TempFolder.cs @@ -15,7 +15,7 @@ public TempFolder(string name) { Path = System.IO.Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), - "Seq.Forwarder.Tests", + "SeqCli.Forwarder.Tests", Session.ToString("n"), name); From 353c7b0108679e5a10982a7db399d7b6e563d424 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 8 May 2024 20:33:01 +1000 Subject: [PATCH 29/51] Hide preview commands behind the --pre flag as Seq does --- src/SeqCli/Cli/CommandAttribute.cs | 2 +- src/SeqCli/Cli/CommandLineHost.cs | 18 +++++--- src/SeqCli/Cli/CommandMetadata.cs | 3 +- .../Cli/Commands/Forwarder/InstallCommand.cs | 2 +- .../Cli/Commands/Forwarder/RestartCommand.cs | 2 +- .../Cli/Commands/Forwarder/RunCommand.cs | 2 +- .../Cli/Commands/Forwarder/StartCommand.cs | 2 +- .../Cli/Commands/Forwarder/StatusCommand.cs | 2 +- .../Cli/Commands/Forwarder/StopCommand.cs | 2 +- .../Cli/Commands/Forwarder/TruncateCommand.cs | 2 +- .../Commands/Forwarder/UninstallCommand.cs | 2 +- src/SeqCli/Cli/Commands/HelpCommand.cs | 38 ++++++++++------- .../Help/MarkdownHelpTestCase.cs | 2 + test/SeqCli.Tests/Cli/CommandLineHostTests.cs | 41 ++++++++++++------- test/SeqCli.Tests/Support/ActionCommand.cs | 12 ++++++ 15 files changed, 87 insertions(+), 45 deletions(-) create mode 100644 test/SeqCli.Tests/Support/ActionCommand.cs diff --git a/src/SeqCli/Cli/CommandAttribute.cs b/src/SeqCli/Cli/CommandAttribute.cs index 775e03a5..a41f5422 100644 --- a/src/SeqCli/Cli/CommandAttribute.cs +++ b/src/SeqCli/Cli/CommandAttribute.cs @@ -22,8 +22,8 @@ public class CommandAttribute : Attribute, ICommandMetadata public string Name { get; } public string? SubCommand { get; } public string HelpText { get; } - public string? Example { get; set; } + public bool IsPreview { get; set; } public CommandAttribute(string name, string helpText) { diff --git a/src/SeqCli/Cli/CommandLineHost.cs b/src/SeqCli/Cli/CommandLineHost.cs index e5c686fc..407282ef 100644 --- a/src/SeqCli/Cli/CommandLineHost.cs +++ b/src/SeqCli/Cli/CommandLineHost.cs @@ -39,19 +39,25 @@ public async Task Run(string[] args, LoggingLevelSwitch levelSwitch) if (args.Length > 0) { + const string prereleaseArg = "--pre", verboseArg = "--verbose"; + var norm = args[0].ToLowerInvariant(); var subCommandNorm = args.Length > 1 && !args[1].Contains('-') ? args[1].ToLowerInvariant() : null; - + + var pre = args.Any(a => a == prereleaseArg); + var cmd = _availableCommands.SingleOrDefault(c => - c.Metadata.Name == norm && (c.Metadata.SubCommand == subCommandNorm || c.Metadata.SubCommand == null)); + (!c.Metadata.IsPreview || pre) && + c.Metadata.Name == norm && + (c.Metadata.SubCommand == subCommandNorm || c.Metadata.SubCommand == null)); if (cmd != null) { var amountToSkip = cmd.Metadata.SubCommand == null ? 1 : 2; - var commandSpecificArgs = args.Skip(amountToSkip).ToArray(); + var commandSpecificArgs = args.Skip(amountToSkip).Where(arg => arg != prereleaseArg).ToArray(); - var verboseArg = commandSpecificArgs.FirstOrDefault(arg => arg == "--verbose"); - if (verboseArg != null) + var verbose = commandSpecificArgs.Any(arg => arg == verboseArg); + if (verbose) { levelSwitch.MinimumLevel = LogEventLevel.Information; commandSpecificArgs = commandSpecificArgs.Where(arg => arg != verboseArg).ToArray(); @@ -63,6 +69,6 @@ public async Task Run(string[] args, LoggingLevelSwitch levelSwitch) Console.WriteLine($"Usage: {name} []"); Console.WriteLine($"Type `{name} help` for available commands"); - return -1; + return 1; } } \ No newline at end of file diff --git a/src/SeqCli/Cli/CommandMetadata.cs b/src/SeqCli/Cli/CommandMetadata.cs index e691cce7..49b18d89 100644 --- a/src/SeqCli/Cli/CommandMetadata.cs +++ b/src/SeqCli/Cli/CommandMetadata.cs @@ -20,4 +20,5 @@ public class CommandMetadata : ICommandMetadata public string? SubCommand { get; set; } public string HelpText { get; set; } = null!; public string? Example { get; set; } -} \ No newline at end of file + public bool IsPreview { get; set; } +} diff --git a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs index 7d709715..b3bb1c3d 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs @@ -35,7 +35,7 @@ namespace SeqCli.Forwarder.Cli.Commands { - [Command("forwarder", "install", "Install the forwarder as a Windows service")] + [Command("forwarder", "install", "Install the forwarder as a Windows service", IsPreview = true)] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class InstallCommand : Command { diff --git a/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs index 63008cb0..65a6a6b9 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs @@ -26,7 +26,7 @@ namespace SeqCli.Forwarder.Cli.Commands { - [Command("forwarder", "restart", "Restart the forwarder Windows service")] + [Command("forwarder", "restart", "Restart the forwarder Windows service", IsPreview = true)] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class RestartCommand : Command { diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index 9a2a1b32..4e69cebd 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -45,7 +45,7 @@ namespace SeqCli.Cli.Commands.Forwarder; -[Command("forwarder", "run", "Listen on an HTTP endpoint and forward ingested logs to Seq")] +[Command("forwarder", "run", "Listen on an HTTP endpoint and forward ingested logs to Seq", IsPreview = true)] class RunCommand : Command { readonly StoragePathFeature _storagePath; diff --git a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs index 98ee92f8..66f859fc 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs @@ -24,7 +24,7 @@ namespace SeqCli.Forwarder.Cli.Commands { - [Command("forwarder", "start", "Start the forwarder Windows service")] + [Command("forwarder", "start", "Start the forwarder Windows service", IsPreview = true)] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class StartCommand : Command { diff --git a/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs index 3d0073b1..cd5fa6f6 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs @@ -24,7 +24,7 @@ namespace SeqCli.Forwarder.Cli.Commands { - [Command("forwarder", "status", "Show the status of the forwarder Windows service")] + [Command("forwarder", "status", "Show the status of the forwarder Windows service", IsPreview = true)] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class StatusCommand : Command { diff --git a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs index 88d7db6b..955c550c 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs @@ -24,7 +24,7 @@ namespace SeqCli.Forwarder.Cli.Commands { - [Command("forwarder", "stop", "Stop the forwarder Windows service")] + [Command("forwarder", "stop", "Stop the forwarder Windows service", IsPreview = true)] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] class StopCommand : Command { diff --git a/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs index 844eb9ee..bf50a2ae 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs @@ -19,7 +19,7 @@ namespace SeqCli.Cli.Commands.Forwarder; -[Command("forwarder", "truncate", "Empty the forwarder's persistent log buffer")] +[Command("forwarder", "truncate", "Empty the forwarder's persistent log buffer", IsPreview = true)] class TruncateCommand : Command { readonly StoragePathFeature _storagePath; diff --git a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs index 224cc94c..20bfbcdd 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs @@ -24,7 +24,7 @@ namespace SeqCli.Forwarder.Cli.Commands { - [Command("forwarder", "uninstall", "Uninstall the forwarder Windows service")] + [Command("forwarder", "uninstall", "Uninstall the forwarder Windows service", IsPreview = true)] class UninstallCommand : Command { protected override Task Run() diff --git a/src/SeqCli/Cli/Commands/HelpCommand.cs b/src/SeqCli/Cli/Commands/HelpCommand.cs index 026cfcac..3aab532b 100644 --- a/src/SeqCli/Cli/Commands/HelpCommand.cs +++ b/src/SeqCli/Cli/Commands/HelpCommand.cs @@ -18,23 +18,31 @@ using System.Reflection; using System.Threading.Tasks; using Autofac.Features.Metadata; +using CommandList = System.Collections.Generic.List, SeqCli.Cli.CommandMetadata>>; namespace SeqCli.Cli.Commands; [Command("help", "Show information about available commands", Example = "seqcli help search")] class HelpCommand : Command { - readonly List, CommandMetadata>> _orderedCommands; - bool _markdown; + readonly IEnumerable, CommandMetadata>> _availableCommands; + bool _markdown, _pre; public HelpCommand(IEnumerable, CommandMetadata>> availableCommands) { + _availableCommands = availableCommands; + Options.Add("pre", "Show preview commands", _ => _pre = true); Options.Add("m|markdown", "Generate markdown for use in documentation", _ => _markdown = true); - _orderedCommands = availableCommands.OrderBy(c => c.Metadata.Name).ThenBy(c => c.Metadata.SubCommand).ToList(); } protected override Task Run(string[] unrecognized) { + var orderedCommands = _availableCommands + .Where(c => !c.Metadata.IsPreview || _pre) + .OrderBy(c => c.Metadata.Name) + .ThenBy(c => c.Metadata.SubCommand) + .ToList(); + var ea = Assembly.GetEntryAssembly(); // ReSharper disable once PossibleNullReferenceException var name = ea!.GetName().Name!; @@ -44,7 +52,7 @@ protected override Task Run(string[] unrecognized) if (unrecognized.Length != 0) return base.Run(unrecognized); - PrintMarkdownHelp(name); + PrintMarkdownHelp(name, orderedCommands); return Task.FromResult(0); } @@ -53,7 +61,7 @@ protected override Task Run(string[] unrecognized) { topLevelCommand = unrecognized[0].ToLowerInvariant(); var subCommand = unrecognized.Length > 1 && !unrecognized[1].Contains("-") ? unrecognized[1] : null; - var cmds = _orderedCommands.Where(c => c.Metadata.Name == topLevelCommand && + var cmds = orderedCommands.Where(c => c.Metadata.Name == topLevelCommand && (subCommand == null || subCommand == c.Metadata.SubCommand)).ToArray(); if (cmds.Length == 1 && cmds[0].Metadata.SubCommand == subCommand) @@ -79,15 +87,15 @@ protected override Task Run(string[] unrecognized) } } - if (topLevelCommand != null && _orderedCommands.Any(a => a.Metadata.Name == topLevelCommand)) - PrintHelp(name, topLevelCommand); + if (topLevelCommand != null && orderedCommands.Any(a => a.Metadata.Name == topLevelCommand)) + PrintHelp(name, topLevelCommand, orderedCommands); else - PrintHelp(name); + PrintHelp(name, orderedCommands); return Task.FromResult(0); } - void PrintMarkdownHelp(string executableName) + static void PrintMarkdownHelp(string executableName, CommandList orderedCommands) { Console.WriteLine("## Commands"); Console.WriteLine(); @@ -101,7 +109,7 @@ void PrintMarkdownHelp(string executableName) Console.WriteLine("Available commands:"); Console.WriteLine(); - foreach (var cmd in _orderedCommands.GroupBy(cmd => cmd.Metadata.Name).OrderBy(c => c.Key)) + foreach (var cmd in orderedCommands.GroupBy(cmd => cmd.Metadata.Name).OrderBy(c => c.Key)) { if (cmd.Count() == 1) { @@ -122,7 +130,7 @@ void PrintMarkdownHelp(string executableName) } Console.WriteLine(); - foreach (var cmd in _orderedCommands) + foreach (var cmd in orderedCommands) { if (cmd.Metadata.SubCommand != null) Console.WriteLine($"### `{cmd.Metadata.Name} {cmd.Metadata.SubCommand}`"); @@ -166,14 +174,14 @@ void PrintMarkdownHelp(string executableName) } } - void PrintHelp(string executableName) + static void PrintHelp(string executableName, CommandList orderedCommands) { Console.WriteLine($"Usage: {executableName} []"); Console.WriteLine(); Console.WriteLine("Available commands are:"); var printedGroups = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var avail in _orderedCommands) + foreach (var avail in orderedCommands) { if (avail.Metadata.SubCommand != null) { @@ -193,13 +201,13 @@ void PrintHelp(string executableName) Console.WriteLine($"Type `{executableName} help ` for detailed help"); } - void PrintHelp(string executableName, string topLevelCommand) + static void PrintHelp(string executableName, string topLevelCommand, CommandList orderedCommands) { Console.WriteLine($"Usage: {executableName} {topLevelCommand} []"); Console.WriteLine(); Console.WriteLine("Available sub-commands are:"); - foreach (var avail in _orderedCommands.Where(c => c.Metadata.Name == topLevelCommand)) + foreach (var avail in orderedCommands.Where(c => c.Metadata.Name == topLevelCommand)) { Printing.Define($" {avail.Metadata.SubCommand}", avail.Metadata.HelpText, Console.Out); } diff --git a/test/SeqCli.EndToEnd/Help/MarkdownHelpTestCase.cs b/test/SeqCli.EndToEnd/Help/MarkdownHelpTestCase.cs index bfecf1ff..9177492b 100644 --- a/test/SeqCli.EndToEnd/Help/MarkdownHelpTestCase.cs +++ b/test/SeqCli.EndToEnd/Help/MarkdownHelpTestCase.cs @@ -24,6 +24,8 @@ public Task ExecuteAsync(SeqConnection connection, ILogger logger, CliCommandRun var indexOfTemplateImport = markdown.IndexOf("### `template import`", StringComparison.Ordinal); Assert.NotEqual(indexOfTemplateExport, indexOfTemplateImport); Assert.True(indexOfTemplateExport < indexOfTemplateImport); + + Assert.DoesNotContain("### `forwarder run`", markdown); return Task.CompletedTask; } diff --git a/test/SeqCli.Tests/Cli/CommandLineHostTests.cs b/test/SeqCli.Tests/Cli/CommandLineHostTests.cs index 78bea1e3..ee72b1ef 100644 --- a/test/SeqCli.Tests/Cli/CommandLineHostTests.cs +++ b/test/SeqCli.Tests/Cli/CommandLineHostTests.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Autofac.Features.Metadata; using SeqCli.Cli; +using SeqCli.Tests.Support; using Serilog.Core; using Serilog.Events; using Xunit; @@ -13,7 +14,7 @@ namespace SeqCli.Tests.Cli; public class CommandLineHostTests { [Fact] - public async Task CheckCommandLineHostPicksCorrectCommand() + public async Task CommandLineHostPicksCorrectCommand() { var executed = new List(); var availableCommands = new List, CommandMetadata>> @@ -28,27 +29,47 @@ public async Task CheckCommandLineHostPicksCorrectCommand() var commandLineHost = new CommandLineHost(availableCommands); await commandLineHost.Run(["test"],new LoggingLevelSwitch()); - Assert.Equal("test", executed.First()); + Assert.Equal("test", executed.Single()); + } + + [Fact] + public async Task PrereleaseCommandsAreIgnoredWithoutFlag() + { + var executed = new List(); + var availableCommands = new List, CommandMetadata>> + { + new( + new Lazy(() => new ActionCommand(() => executed.Add("test"))), + new CommandMetadata {Name = "test", IsPreview = true}), + }; + var commandLineHost = new CommandLineHost(availableCommands); + var exit = await commandLineHost.Run(["test"],new LoggingLevelSwitch()); + Assert.Equal(1, exit); + Assert.Empty(executed); + + exit = await commandLineHost.Run(["test", "--pre"],new LoggingLevelSwitch()); + Assert.Equal(0, exit); + Assert.Equal("test", executed.Single()); } [Fact] public async Task WhenMoreThanOneSubcommandAndTheUserRunsWithSubcommandEnsurePickedCorrect() { - var commandsRan = new List(); + var executed = new List(); var availableCommands = new List, CommandMetadata>> { new( - new Lazy(() => new ActionCommand(() => commandsRan.Add("test-subcommand1"))), + new Lazy(() => new ActionCommand(() => executed.Add("test-subcommand1"))), new CommandMetadata {Name = "test", SubCommand = "subcommand1"}), new( - new Lazy(() => new ActionCommand(() => commandsRan.Add("test-subcommand2"))), + new Lazy(() => new ActionCommand(() => executed.Add("test-subcommand2"))), new CommandMetadata {Name = "test", SubCommand = "subcommand2"}) }; var commandLineHost = new CommandLineHost(availableCommands); await commandLineHost.Run(["test", "subcommand2"], new LoggingLevelSwitch()); - Assert.Equal("test-subcommand2", commandsRan.First()); + Assert.Equal("test-subcommand2", executed.First()); } [Fact] @@ -70,12 +91,4 @@ public async Task VerboseOptionSetsLoggingLevelToInformation() Assert.Equal(LogEventLevel.Information, levelSwitch.MinimumLevel); } - - class ActionCommand : Command - { - public ActionCommand(Action action) - { - action.Invoke(); - } - } } \ No newline at end of file diff --git a/test/SeqCli.Tests/Support/ActionCommand.cs b/test/SeqCli.Tests/Support/ActionCommand.cs new file mode 100644 index 00000000..bf7ae4c9 --- /dev/null +++ b/test/SeqCli.Tests/Support/ActionCommand.cs @@ -0,0 +1,12 @@ +using System; +using SeqCli.Cli; + +namespace SeqCli.Tests.Support; + +class ActionCommand : Command +{ + public ActionCommand(Action action) + { + action.Invoke(); + } +} \ No newline at end of file From 18673f4e7b006062fa2fd90ecf1b57dbeb505cbb Mon Sep 17 00:00:00 2001 From: KodrAus Date: Thu, 9 May 2024 08:47:50 +1000 Subject: [PATCH 30/51] fix up build on Windows --- src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs | 3 --- src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs | 1 + src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs | 1 - src/SeqCli/Cli/Features/StoragePathFeature.cs | 6 +++++- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs index b3bb1c3d..c6bbfb4c 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs @@ -23,13 +23,10 @@ using System.Threading.Tasks; using SeqCli.Forwarder.Cli.Features; using SeqCli.Forwarder.Util; -using SeqCli; using SeqCli.Cli; using SeqCli.Cli.Features; using SeqCli.Config; -using SeqCli.Config.Forwarder; using SeqCli.Forwarder.ServiceProcess; -using SeqCli.Forwarder.Util; // ReSharper disable once ClassNeverInstantiated.Global diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index 4e69cebd..fb9af0a3 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -38,6 +38,7 @@ using Serilog.Formatting.Compact; #if WINDOWS +using System.Security.Cryptography.X509Certificates; using SeqCli.Forwarder.ServiceProcess; #endif diff --git a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs index 20bfbcdd..50845d26 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs @@ -20,7 +20,6 @@ using SeqCli.Forwarder.Util; using SeqCli.Cli; using SeqCli.Forwarder.ServiceProcess; -using SeqCli.Forwarder.Util; namespace SeqCli.Forwarder.Cli.Commands { diff --git a/src/SeqCli/Cli/Features/StoragePathFeature.cs b/src/SeqCli/Cli/Features/StoragePathFeature.cs index 809a62b6..b129f796 100644 --- a/src/SeqCli/Cli/Features/StoragePathFeature.cs +++ b/src/SeqCli/Cli/Features/StoragePathFeature.cs @@ -1,6 +1,10 @@ using System; using System.IO; +#if WINDOWS +using SeqCli.Forwarder.ServiceProcess; +#endif + namespace SeqCli.Cli.Features; class StoragePathFeature : CommandFeature @@ -48,7 +52,7 @@ static string GetDefaultStorageRoot() static string? TryQueryInstalledStorageRoot() { #if WINDOWS - if (SeqCli.Forwarder.Util.ServiceConfiguration.GetServiceStoragePath( + if (Forwarder.Util.ServiceConfiguration.GetServiceStoragePath( SeqCliForwarderWindowsService.WindowsServiceName, out var storage)) return storage; #endif From dc27108d9bf0ea42a094a6fdfa52956560653043 Mon Sep 17 00:00:00 2001 From: KodrAus Date: Thu, 9 May 2024 09:03:14 +1000 Subject: [PATCH 31/51] add a durable buffer implementation for forwarder --- .../LogBuffer.cs => Channel/LogChannel.cs} | 12 +- .../Forwarder/Channel/LogChannelEntry.cs | 6 + .../LogChannelMap.cs} | 10 +- .../Filesystem/EmptyStoreFileReader.cs | 29 ++ .../Forwarder/Filesystem/StoreDirectory.cs | 91 +++++ src/SeqCli/Forwarder/Filesystem/StoreFile.cs | 65 ++++ .../Forwarder/Filesystem/StoreFileAppender.cs | 37 ++ .../Forwarder/Filesystem/StoreFileReader.cs | 27 ++ .../Filesystem/System/SystemStoreDirectory.cs | 125 +++++++ .../Filesystem/System/SystemStoreFile.cs | 122 +++++++ .../System/SystemStoreFileAppender.cs | 58 ++++ .../System/SystemStoreFileReader.cs | 52 +++ .../Forwarder/Filesystem/System/Unix/Libc.cs | 29 ++ src/SeqCli/Forwarder/ForwarderModule.cs | 4 +- src/SeqCli/Forwarder/Storage/Bookmark.cs | 144 ++++++++ src/SeqCli/Forwarder/Storage/BookmarkName.cs | 56 +++ src/SeqCli/Forwarder/Storage/BookmarkValue.cs | 54 +++ .../Forwarder/Storage/BufferAppender.cs | 164 +++++++++ .../Forwarder/Storage/BufferAppenderChunk.cs | 28 ++ src/SeqCli/Forwarder/Storage/BufferReader.cs | 325 ++++++++++++++++++ .../Forwarder/Storage/BufferReaderBatch.cs | 50 +++ .../Forwarder/Storage/BufferReaderChunk.cs | 56 +++ .../Storage/BufferReaderChunkHead.cs | 23 ++ .../Forwarder/Storage/BufferReaderHead.cs | 20 ++ src/SeqCli/Forwarder/Storage/ChunkName.cs | 56 +++ src/SeqCli/Forwarder/Storage/Identifier.cs | 60 ++++ .../Forwarder/Storage/LogBufferEntry.cs | 6 - .../Forwarder/Web/Api/IngestionEndpoints.cs | 12 +- .../Forwarder/Web/Host/ServerService.cs | 10 +- src/SeqCli/SeqCli.csproj | 1 + .../Filesystem/InMemoryStoreDirectory.cs | 50 +++ .../Forwarder/Filesystem/InMemoryStoreFile.cs | 38 ++ .../Filesystem/InMemoryStoreFileAppender.cs | 39 +++ .../Filesystem/InMemoryStoreFileReader.cs | 29 ++ .../Forwarder/Storage/BookmarkTests.cs | 83 +++++ .../Forwarder/Storage/BufferTests.cs | 283 +++++++++++++++ .../Forwarder/Storage/IdentifierTests.cs | 31 ++ test/SeqCli.Tests/SeqCli.Tests.csproj | 1 + 38 files changed, 2256 insertions(+), 30 deletions(-) rename src/SeqCli/Forwarder/{Storage/LogBuffer.cs => Channel/LogChannel.cs} (78%) create mode 100644 src/SeqCli/Forwarder/Channel/LogChannelEntry.cs rename src/SeqCli/Forwarder/{Storage/LogBufferMap.cs => Channel/LogChannelMap.cs} (50%) create mode 100644 src/SeqCli/Forwarder/Filesystem/EmptyStoreFileReader.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/StoreFile.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/StoreFileAppender.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/StoreFileReader.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/System/SystemStoreFile.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileAppender.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileReader.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/System/Unix/Libc.cs create mode 100644 src/SeqCli/Forwarder/Storage/Bookmark.cs create mode 100644 src/SeqCli/Forwarder/Storage/BookmarkName.cs create mode 100644 src/SeqCli/Forwarder/Storage/BookmarkValue.cs create mode 100644 src/SeqCli/Forwarder/Storage/BufferAppender.cs create mode 100644 src/SeqCli/Forwarder/Storage/BufferAppenderChunk.cs create mode 100644 src/SeqCli/Forwarder/Storage/BufferReader.cs create mode 100644 src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs create mode 100644 src/SeqCli/Forwarder/Storage/BufferReaderChunk.cs create mode 100644 src/SeqCli/Forwarder/Storage/BufferReaderChunkHead.cs create mode 100644 src/SeqCli/Forwarder/Storage/BufferReaderHead.cs create mode 100644 src/SeqCli/Forwarder/Storage/ChunkName.cs create mode 100644 src/SeqCli/Forwarder/Storage/Identifier.cs delete mode 100644 src/SeqCli/Forwarder/Storage/LogBufferEntry.cs create mode 100644 test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreDirectory.cs create mode 100644 test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs create mode 100644 test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileAppender.cs create mode 100644 test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileReader.cs create mode 100644 test/SeqCli.Tests/Forwarder/Storage/BookmarkTests.cs create mode 100644 test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs create mode 100644 test/SeqCli.Tests/Forwarder/Storage/IdentifierTests.cs diff --git a/src/SeqCli/Forwarder/Storage/LogBuffer.cs b/src/SeqCli/Forwarder/Channel/LogChannel.cs similarity index 78% rename from src/SeqCli/Forwarder/Storage/LogBuffer.cs rename to src/SeqCli/Forwarder/Channel/LogChannel.cs index 25bde0d9..4a02585b 100644 --- a/src/SeqCli/Forwarder/Storage/LogBuffer.cs +++ b/src/SeqCli/Forwarder/Channel/LogChannel.cs @@ -3,13 +3,13 @@ using System.Threading.Channels; using System.Threading.Tasks; -namespace SeqCli.Forwarder.Storage; +namespace SeqCli.Forwarder.Channel; -class LogBuffer +class LogChannel { - public LogBuffer(Func write, CancellationToken cancellationToken) + public LogChannel(Func write, CancellationToken cancellationToken) { - var channel = Channel.CreateBounded(new BoundedChannelOptions(5) + var channel = System.Threading.Channels.Channel.CreateBounded(new BoundedChannelOptions(5) { SingleReader = false, SingleWriter = true, @@ -35,7 +35,7 @@ public LogBuffer(Func write, CancellationToken cancella }, cancellationToken: _shutdownTokenSource.Token); } - readonly ChannelWriter _writer; + readonly ChannelWriter _writer; readonly Task _worker; readonly CancellationTokenSource _shutdownTokenSource; @@ -44,7 +44,7 @@ public async Task WriteAsync(byte[] storage, Range range, CancellationToken canc var tcs = new TaskCompletionSource(); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownTokenSource.Token); - await _writer.WriteAsync(new LogBufferEntry(storage, range, tcs), cts.Token); + await _writer.WriteAsync(new LogChannelEntry(storage, range, tcs), cts.Token); await tcs.Task; } diff --git a/src/SeqCli/Forwarder/Channel/LogChannelEntry.cs b/src/SeqCli/Forwarder/Channel/LogChannelEntry.cs new file mode 100644 index 00000000..866d135a --- /dev/null +++ b/src/SeqCli/Forwarder/Channel/LogChannelEntry.cs @@ -0,0 +1,6 @@ +using System; +using System.Threading.Tasks; + +namespace SeqCli.Forwarder.Channel; + +public readonly record struct LogChannelEntry(byte[] Storage, Range Range, TaskCompletionSource Completion); diff --git a/src/SeqCli/Forwarder/Storage/LogBufferMap.cs b/src/SeqCli/Forwarder/Channel/LogChannelMap.cs similarity index 50% rename from src/SeqCli/Forwarder/Storage/LogBufferMap.cs rename to src/SeqCli/Forwarder/Channel/LogChannelMap.cs index d9601814..3bb24e90 100644 --- a/src/SeqCli/Forwarder/Storage/LogBufferMap.cs +++ b/src/SeqCli/Forwarder/Channel/LogChannelMap.cs @@ -2,18 +2,18 @@ using System.Threading.Tasks; using Serilog; -namespace SeqCli.Forwarder.Storage; +namespace SeqCli.Forwarder.Channel; -class LogBufferMap +class LogChannelMap { - public LogBufferMap() + public LogChannelMap() { } - public LogBuffer Get(string? apiKey) + public LogChannel Get(string? apiKey) { - return new LogBuffer(async (c) => await Task.Delay(TimeSpan.FromSeconds(1), c), default); + return new LogChannel(async (c) => await Task.Delay(TimeSpan.FromSeconds(1), c), default); } public Task StopAsync() diff --git a/src/SeqCli/Forwarder/Filesystem/EmptyStoreFileReader.cs b/src/SeqCli/Forwarder/Filesystem/EmptyStoreFileReader.cs new file mode 100644 index 00000000..0410e23a --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/EmptyStoreFileReader.cs @@ -0,0 +1,29 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace SeqCli.Forwarder.Filesystem; + +public sealed class EmptyStoreFileReader : StoreFileReader +{ + public override void Dispose() + { + } + + public override long CopyTo(Span buffer, long from = 0, long? length = null) + { + return 0; + } +} diff --git a/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs b/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs new file mode 100644 index 00000000..e9503162 --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs @@ -0,0 +1,91 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; + +namespace SeqCli.Forwarder.Filesystem; + +/// +/// A container of s and their names. +/// +public abstract class StoreDirectory +{ + /// + /// Create a new file with the given name, linking it into the filesystem. + /// + public abstract StoreFile Create(string name); + + public virtual (string, StoreFile) CreateTemporary() + { + var tmpName = $"rc{Guid.NewGuid():N}.tmp"; + return (tmpName, Create(tmpName)); + } + + /// + /// Delete a file with the given name, returning whether the file was deleted. + /// + public abstract bool TryDelete(string name); + + /// + /// Atomically replace the contents of one file with another, creating it if it doesn't exist and deleting the other. + /// + public abstract StoreFile Replace(string toReplace, string replaceWith); + + /// + /// Atomically replace the contents of a file. + /// + public virtual StoreFile ReplaceContents(string name, Span contents, bool sync = true) + { + var (tmpName, tmpFile) = CreateTemporary(); + + try + { + if (!tmpFile.TryOpenAppend(out var opened)) + throw new Exception("Failed to write to a temporary file that was just created."); + + using var writer = opened; + writer.Append(contents); + writer.Commit(); + + if (sync) writer.Sync(); + } + catch + { + TryDelete(tmpName); + throw; + } + + return Replace(name, tmpName); + } + + /// + /// List all files in unspecified order. + /// + public abstract IEnumerable<(string Name, StoreFile File)> List(Func predicate); + + /// + /// Try get a file by name. + /// + public virtual bool TryGet(string name, [NotNullWhen(true)] out StoreFile? file) + { + file = List(n => n == name) + .Select(p => p.File) + .FirstOrDefault(); + + return file != null; + } +} diff --git a/src/SeqCli/Forwarder/Filesystem/StoreFile.cs b/src/SeqCli/Forwarder/Filesystem/StoreFile.cs new file mode 100644 index 00000000..9e998523 --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/StoreFile.cs @@ -0,0 +1,65 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics.CodeAnalysis; + +namespace SeqCli.Forwarder.Filesystem; + +public abstract class StoreFile +{ + /// + /// Get the length of this file. + /// + /// + /// True if the length was read, false if the file is invalid. + /// + public abstract bool TryGetLength([NotNullWhen(true)] out long? length); + + /// + /// Read the complete contents of this file. + /// + /// + /// The number of bytes copied. + /// + public virtual long CopyContentsTo(Span buffer) + { + if (!TryGetLength(out var length)) throw new Exception("Failed to get the length of a file."); + + if (!TryOpenRead(length.Value, out var opened)) throw new Exception("Failed to open a reader to a file."); + + using var reader = opened; + return reader.CopyTo(buffer); + } + + /// + /// Try open a reader to the file. + /// + /// + /// True if the file was opened for reading, false if the file is invalid. + /// + public abstract bool TryOpenRead(long length, [NotNullWhen(true)] out StoreFileReader? reader); + + /// + /// Open a writer to the file. + /// + /// + /// Only a single writer to a file should be open at a given time. + /// Overlapping writers may result in data corruption. + /// + /// + /// True if the file was opened for writing, false if the file is invalid. + /// + public abstract bool TryOpenAppend([NotNullWhen(true)] out StoreFileAppender? appender); +} diff --git a/src/SeqCli/Forwarder/Filesystem/StoreFileAppender.cs b/src/SeqCli/Forwarder/Filesystem/StoreFileAppender.cs new file mode 100644 index 00000000..8e04dd91 --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/StoreFileAppender.cs @@ -0,0 +1,37 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace SeqCli.Forwarder.Filesystem; + +public abstract class StoreFileAppender : IDisposable +{ + public abstract void Dispose(); + + /// + /// Append the given data to the end of the file. + /// + public abstract void Append(Span data); + + /// + /// Commit all appended data to underlying storage. + /// + public abstract long Commit(); + + /// + /// Durably sync committed data to underlying storage. + /// + public abstract void Sync(); +} diff --git a/src/SeqCli/Forwarder/Filesystem/StoreFileReader.cs b/src/SeqCli/Forwarder/Filesystem/StoreFileReader.cs new file mode 100644 index 00000000..d98feaad --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/StoreFileReader.cs @@ -0,0 +1,27 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace SeqCli.Forwarder.Filesystem; + +public abstract class StoreFileReader : IDisposable +{ + public abstract void Dispose(); + + /// + /// Copy the complete contents of the reader to the given buffer. + /// + public abstract long CopyTo(Span buffer, long from = 0, long? length = null); +} diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs new file mode 100644 index 00000000..be27c3d4 --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs @@ -0,0 +1,125 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; +using SeqCli.Forwarder.Filesystem.System.Unix; + +namespace SeqCli.Forwarder.Filesystem.System; + +public sealed class SystemStoreDirectory : StoreDirectory +{ + readonly string _directoryPath; + + public SystemStoreDirectory(string path) + { + _directoryPath = Path.GetFullPath(path); + + if (!Directory.Exists(_directoryPath)) Directory.CreateDirectory(_directoryPath); + } + + public override SystemStoreFile Create(string name) + { + var filePath = Path.Combine(_directoryPath, name); + using var _ = File.OpenHandle(filePath, FileMode.Create, FileAccess.ReadWrite, + FileShare.ReadWrite | FileShare.Delete); + Dirsync(_directoryPath); + + return new SystemStoreFile(filePath); + } + + public override (string, StoreFile) CreateTemporary() + { + // Temporary files are still created in the same directory + // This is necessary for renames to be atomic on some filesystems + var tmpName = $"rc{Guid.NewGuid():N}.tmp"; + + var filePath = Path.Combine(_directoryPath, tmpName); + using var _ = File.OpenHandle(filePath, FileMode.CreateNew, FileAccess.ReadWrite, + FileShare.ReadWrite | FileShare.Delete, FileOptions.DeleteOnClose); + + return (tmpName, new SystemStoreFile(filePath)); + } + + public override bool TryDelete(string name) + { + var filePath = Path.Combine(_directoryPath, name); + + try + { + File.Delete(filePath); + return true; + } + catch (IOException) + { + return false; + } + } + + public override SystemStoreFile Replace(string toReplace, string replaceWith) + { + var filePath = Path.Combine(_directoryPath, toReplace); + + File.Replace(Path.Combine(_directoryPath, replaceWith), filePath, null); + + return new SystemStoreFile(filePath); + } + + public override StoreFile ReplaceContents(string name, Span contents, bool sync = true) + { + var filePath = Path.Combine(_directoryPath, name); + + using var file = File.Open(filePath, FileMode.OpenOrCreate, FileAccess.ReadWrite, + FileShare.ReadWrite | FileShare.Delete); + + // NOTE: This will be atomic if: + // 1. The incoming contents are larger or equal in size to the length of the file + // 2. The incoming contents are page sized or smaller + file.Position = 0; + file.Write(contents); + + if (sync) file.Flush(true); + + return new SystemStoreFile(filePath); + } + + public override IEnumerable<(string Name, StoreFile File)> List(Func predicate) + { + foreach (var filePath in Directory.EnumerateFiles(_directoryPath)) + { + var name = Path.GetFileName(filePath); + + if (!predicate(name)) continue; + + yield return (name, new SystemStoreFile(filePath)); + } + } + + static void Dirsync(string directoryPath) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return; + + var dir = Libc.open(directoryPath, 0); + if (dir == -1) return; + + // NOTE: directory syncing here is best-effort + // If it fails for any reason we simply carry on +#pragma warning disable CA1806 + Libc.fsync(dir); + Libc.close(dir); +#pragma warning restore CA1806 + } +} diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFile.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFile.cs new file mode 100644 index 00000000..361b6741 --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFile.cs @@ -0,0 +1,122 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.IO.MemoryMappedFiles; + +namespace SeqCli.Forwarder.Filesystem.System; + +public sealed class SystemStoreFile : StoreFile +{ + static readonly FileStreamOptions AppendOptions = new() + { + Mode = FileMode.Append, + Access = FileAccess.Write, + Share = FileShare.ReadWrite | FileShare.Delete + }; + + readonly string _filePath; + + internal SystemStoreFile(string filePath) + { + _filePath = filePath; + } + + public override bool TryGetLength([NotNullWhen(true)] out long? length) + { + try + { + length = new FileInfo(_filePath).Length; + return true; + } + catch (IOException) + { + length = null; + return false; + } + } + + public override bool TryOpenRead(long length, [NotNullWhen(true)] out StoreFileReader? reader) + { + MemoryMappedFile? disposeMmap = null; + MemoryMappedViewAccessor? disposeAccessor = null; + + // If the requested length is empty then just return a dummy reader + if (length == 0) + { + reader = new EmptyStoreFileReader(); + return true; + } + + try + { + using var file = File.OpenHandle(_filePath, FileMode.OpenOrCreate, FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete, FileOptions.SequentialScan); + + disposeMmap = MemoryMappedFile.CreateFromFile(file, null, 0, MemoryMappedFileAccess.Read, + HandleInheritability.None, + false); + disposeAccessor = disposeMmap.CreateViewAccessor(0, length, MemoryMappedFileAccess.Read); + + var mmap = disposeMmap; + var accessor = disposeAccessor; + + disposeMmap = null; + disposeAccessor = null; + + reader = new SystemStoreFileReader(mmap, accessor, length); + return true; + } + // Thrown if the file length is 0 + catch (ArgumentException) + { + reader = null; + return false; + } + // Thrown if the file is truncated while creating an accessor + catch (UnauthorizedAccessException) + { + reader = null; + return false; + } + // Thrown if the file is deleted + catch (IOException) + { + reader = null; + return false; + } + finally + { + disposeMmap?.Dispose(); + disposeAccessor?.Dispose(); + } + } + + public override bool TryOpenAppend([NotNullWhen(true)] out StoreFileAppender? appender) + { + try + { + var file = File.Open(_filePath, AppendOptions); + appender = new SystemStoreFileAppender(file); + return true; + } + catch (IOException) + { + appender = null; + return false; + } + } +} diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileAppender.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileAppender.cs new file mode 100644 index 00000000..b64b9d26 --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileAppender.cs @@ -0,0 +1,58 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; + +namespace SeqCli.Forwarder.Filesystem.System; + +public sealed class SystemStoreFileAppender : StoreFileAppender +{ + readonly FileStream _file; + long _initialLength; + long _written; + + internal SystemStoreFileAppender(FileStream file) + { + _file = file; + _initialLength = _file.Length; + _written = 0; + } + + public override void Append(Span data) + { + _written += data.Length; + _file.Write(data); + } + + public override long Commit() + { + var writeHead = _initialLength + _written; + + _initialLength = writeHead; + _written = 0; + + return writeHead; + } + + public override void Sync() + { + _file.Flush(true); + } + + public override void Dispose() + { + _file.Dispose(); + } +} diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileReader.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileReader.cs new file mode 100644 index 00000000..23e95101 --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileReader.cs @@ -0,0 +1,52 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO.MemoryMappedFiles; + +namespace SeqCli.Forwarder.Filesystem.System; + +public sealed class SystemStoreFileReader : StoreFileReader +{ + readonly MemoryMappedViewAccessor _accessor; + readonly MemoryMappedFile _file; + readonly long _length; + + internal SystemStoreFileReader(MemoryMappedFile file, MemoryMappedViewAccessor accessor, long length) + { + _file = file; + + _accessor = accessor; + _length = length; + } + + public override long CopyTo(Span buffer, long from = 0, long? length = null) + { + unsafe + { + var ptr = (byte*)_accessor.SafeMemoryMappedViewHandle.DangerousGetHandle(); + var memmap = new Span(ptr + from, (int)(length ?? _length)); + + memmap.CopyTo(buffer); + + return memmap.Length; + } + } + + public override void Dispose() + { + _accessor.Dispose(); + _file.Dispose(); + } +} diff --git a/src/SeqCli/Forwarder/Filesystem/System/Unix/Libc.cs b/src/SeqCli/Forwarder/Filesystem/System/Unix/Libc.cs new file mode 100644 index 00000000..c9edc557 --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/System/Unix/Libc.cs @@ -0,0 +1,29 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Runtime.InteropServices; + +namespace SeqCli.Forwarder.Filesystem.System.Unix; + +static class Libc +{ + [DllImport("libc")] + public static extern int open(string path, int flags); + + [DllImport("libc")] + public static extern int close(int fd); + + [DllImport("libc")] + public static extern int fsync(int fd); +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 9f75f2dc..03d24974 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -17,7 +17,7 @@ using System.Threading; using Autofac; using SeqCli.Config; -using SeqCli.Forwarder.Storage; +using SeqCli.Forwarder.Channel; using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; using Serilog.Formatting.Display; @@ -38,7 +38,7 @@ public ForwarderModule(string bufferPath, SeqCliConfig config) protected override void Load(ContainerBuilder builder) { builder.RegisterType().SingleInstance(); - builder.RegisterType().SingleInstance(); + builder.RegisterType().SingleInstance(); builder.RegisterType().As(); builder.RegisterType().As(); diff --git a/src/SeqCli/Forwarder/Storage/Bookmark.cs b/src/SeqCli/Forwarder/Storage/Bookmark.cs new file mode 100644 index 00000000..f3ec3ea4 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/Bookmark.cs @@ -0,0 +1,144 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using SeqCli.Forwarder.Filesystem; +using Path = System.IO.Path; + +namespace SeqCli.Forwarder.Storage; + +/// +/// A durable bookmark of progress processing buffers. +/// +public sealed class Bookmark +{ + readonly StoreDirectory _storeDirectory; + + readonly object _sync = new(); + BookmarkName _name; + + BookmarkValue? _value; + + Bookmark(StoreDirectory storeDirectory, BookmarkName name, BookmarkValue? value) + { + _storeDirectory = storeDirectory; + _name = name; + _value = value; + } + + public static Bookmark Open(StoreDirectory storeDirectory) + { + var (name, value) = Read(storeDirectory); + + return new Bookmark(storeDirectory, name, value); + } + + public bool TryGet([NotNullWhen(true)] out BookmarkValue? bookmark) + { + lock (_sync) + { + if (_value != null) + { + bookmark = _value.Value; + return true; + } + + bookmark = null; + return false; + } + } + + public bool TrySet(BookmarkValue value, bool sync = true) + { + lock (_sync) + { + _value = value; + } + + try + { + Write(_storeDirectory, _name, value, sync); + return true; + } + catch (IOException) + { + _name = new BookmarkName(_name.Id + 1); + return false; + } + } + + static void Write(StoreDirectory storeDirectory, BookmarkName name, BookmarkValue value, bool fsync) + { + unsafe + { + Span bookmark = stackalloc byte[16]; + value.EncodeTo(bookmark); + + storeDirectory.ReplaceContents(name.ToString(), bookmark, fsync); + } + } + + static (BookmarkName, BookmarkValue?) Read(StoreDirectory storeDirectory) + { + // NOTE: This method shouldn't throw + var bookmarks = new List<(string, BookmarkName, StoreFile)>(); + + foreach (var (candidateFileName, candidateFile) in storeDirectory + .List(candidateFileName => Path.GetExtension(candidateFileName) is ".bookmark")) + if (BookmarkName.TryParse(candidateFileName, out var parsedBookmarkName)) + bookmarks.Add((candidateFileName, parsedBookmarkName.Value, candidateFile)); + else + // The `.bookmark` file uses an unrecognized naming convention + storeDirectory.TryDelete(candidateFileName); + + switch (bookmarks.Count) + { + // There aren't any bookmarks; return a default one + case 0: + return (new BookmarkName(0), null); + // There are old bookmark values floating around; try delete them again + case > 1: + { + bookmarks.Sort((a, b) => a.Item2.Id.CompareTo(b.Item2.Id)); + + foreach (var (toDelete, _, _) in bookmarks.Take(bookmarks.Count - 1)) + storeDirectory.TryDelete(toDelete); + break; + } + } + + var (fileName, bookmarkName, file) = bookmarks[^1]; + + try + { + unsafe + { + Span bookmark = stackalloc byte[16]; + if (file.CopyContentsTo(bookmark) != 16) throw new Exception("The bookmark is corrupted."); + + return (bookmarkName, BookmarkValue.Decode(bookmark)); + } + } + catch + { + storeDirectory.TryDelete(fileName); + + return (new BookmarkName(bookmarkName.Id + 1), new BookmarkValue()); + } + } +} diff --git a/src/SeqCli/Forwarder/Storage/BookmarkName.cs b/src/SeqCli/Forwarder/Storage/BookmarkName.cs new file mode 100644 index 00000000..76206472 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BookmarkName.cs @@ -0,0 +1,56 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Diagnostics.CodeAnalysis; + +namespace SeqCli.Forwarder.Storage; + +/// +/// A bookmark file name with its incrementing identifier. +/// +public readonly record struct BookmarkName +{ + readonly string _name; + + public readonly ulong Id; + + public BookmarkName(ulong id) + { + Id = id; + _name = Identifier.Format(id, ".bookmark"); + } + + BookmarkName(ulong id, string name) + { + Id = id; + _name = name; + } + + public static bool TryParse(string name, [NotNullWhen(true)] out BookmarkName? parsed) + { + if (Identifier.TryParse(name, ".bookmark", out var id)) + { + parsed = new BookmarkName(id.Value, name); + return true; + } + + parsed = null; + return false; + } + + public override string ToString() + { + return _name; + } +} diff --git a/src/SeqCli/Forwarder/Storage/BookmarkValue.cs b/src/SeqCli/Forwarder/Storage/BookmarkValue.cs new file mode 100644 index 00000000..72716391 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BookmarkValue.cs @@ -0,0 +1,54 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Buffers.Binary; + +namespace SeqCli.Forwarder.Storage; + +/// +/// The in-memory value of a bookmark. +/// +public readonly record struct BookmarkValue(ulong Id, long CommitHead) +{ + public void EncodeTo(Span bookmark) + { + if (bookmark.Length != 16) throw new Exception($"Bookmark values must be 16 bytes (got {bookmark.Length})."); + + BinaryPrimitives.WriteUInt64LittleEndian(bookmark, Id); + BinaryPrimitives.WriteInt64LittleEndian(bookmark[8..], CommitHead); + } + + public byte[] Encode() + { + var buffer = new byte[16]; + EncodeTo(buffer); + + return buffer; + } + + public static BookmarkValue Decode(Span bookmark) + { + if (bookmark.Length != 16) throw new Exception($"Bookmark values must be 16 bytes (got {bookmark.Length})."); + + var id = BinaryPrimitives.ReadUInt64LittleEndian(bookmark); + var commitHead = BinaryPrimitives.ReadInt64LittleEndian(bookmark[8..]); + + return new BookmarkValue + { + Id = id, + CommitHead = commitHead + }; + } +} diff --git a/src/SeqCli/Forwarder/Storage/BufferAppender.cs b/src/SeqCli/Forwarder/Storage/BufferAppender.cs new file mode 100644 index 00000000..b88576fd --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BufferAppender.cs @@ -0,0 +1,164 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using SeqCli.Forwarder.Filesystem; + +namespace SeqCli.Forwarder.Storage; + +/// +/// The write-side of a buffer. +/// +public sealed class BufferAppender : IDisposable +{ + readonly StoreDirectory _storeDirectory; + BufferAppenderChunk? _currentChunk; + + BufferAppender(StoreDirectory storeDirectory) + { + _storeDirectory = storeDirectory; + _currentChunk = null; + } + + public void Dispose() + { + _currentChunk?.Dispose(); + } + + public static BufferAppender Open(StoreDirectory storeDirectory) + { + return new BufferAppender(storeDirectory); + } + + /// + /// Try write a batch. + /// + /// This method does not throw. + /// + /// This method will write the batch into the currently active chunk file unless: + /// + /// 1. The length of the current chunk is greater than or, + /// 2. There is no current chunk, because no writes have been made, or it encountered an IO error previously. + /// + /// If either of these cases is true, then the write will be made to a new chunk file. + /// + /// The newline-delimited data to write. A batch may contain multiple values separated by + /// newlines, but must end on a newline. + /// The file size to roll on. A single batched write may cause the currently + /// active chunk to exceed this size, but a subsequent write will roll over to a new file. + /// The maximum number of chunk files to keep before starting to delete them. This + /// is an optional parameter to use in cases where the reader isn't keeping up with the writer. + /// Whether to explicitly flush the write to disk. + /// True if the write fully succeeded. If this method returns false, it is safe to retry the write, + /// but it may result in duplicate data in the case of partial success. + public bool TryAppend(Span batch, long targetChunkLength, int? maxChunks = null, bool sync = true) + { + if (batch.Length == 0) return true; + + if (batch[^1] != (byte)'\n') throw new Exception("Batches must end with a newline character (\\n)"); + + if (_currentChunk != null) + // Only use the existing chunk if it's writable and shouldn't be rolled over + if (_currentChunk.WriteHead > targetChunkLength) + { + // Run a sync before moving to a new file, just to make sure any + // buffered data makes its way to disk + _currentChunk.Appender.Sync(); + + _currentChunk.Dispose(); + _currentChunk = null; + } + + // If there's no suitable candidate chunk then create a new one + if (_currentChunk == null) + { + var nextChunkId = ReadChunks(_storeDirectory, maxChunks); + + var chunkName = new ChunkName(nextChunkId); + + var chunkFile = _storeDirectory.Create(chunkName.ToString()); + + if (chunkFile.TryOpenAppend(out var opened)) + _currentChunk = new BufferAppenderChunk(opened); + else + return false; + } + + try + { + _currentChunk.Appender.Append(batch); + _currentChunk.Appender.Commit(); + + if (sync) _currentChunk.Appender.Sync(); + + _currentChunk.WriteHead += batch.Length; + + return true; + } + catch (IOException) + { + // Don't try an explicit sync here, because the file already failed to perform IO + + _currentChunk.Dispose(); + _currentChunk = null; + + return false; + } + } + + static ulong ReadChunks(StoreDirectory storeDirectory, int? maxChunks) + { + ulong nextChunkId = 0; + + List? chunks = null; + foreach (var (fileName, _) in storeDirectory.List(candidateName => + Path.GetExtension(candidateName) is ".clef")) + { + if (!ChunkName.TryParse(fileName, out var parsedChunkName)) continue; + + nextChunkId = Math.Max(nextChunkId, parsedChunkName.Value.Id); + + if (maxChunks == null) continue; + + chunks ??= []; + chunks.Add(parsedChunkName.Value); + } + + // Apply retention on the number of chunk files if the reader isn't keeping up + if (chunks != null) + { + ApplyPreWriteRetention(storeDirectory, maxChunks!.Value, chunks); + } + + return nextChunkId + 1; + } + + static void ApplyPreWriteRetention(StoreDirectory storeDirectory, int maxChunks, List unsortedChunks) + { + // We're going to create a new buffer file, so leave room for it if a max is specified + maxChunks = Math.Max(0, maxChunks - 1); + + unsortedChunks.Sort((a, b) => a.Id.CompareTo(b.Id)); + var sortedChunks = unsortedChunks; + + if (sortedChunks.Count > maxChunks) + foreach (var delete in sortedChunks.Take(sortedChunks.Count - maxChunks)) + // This call may fail if a reader is actively holding this file open + // In these cases we let the writer proceed instead of blocking + storeDirectory.TryDelete(delete.ToString()); + } +} diff --git a/src/SeqCli/Forwarder/Storage/BufferAppenderChunk.cs b/src/SeqCli/Forwarder/Storage/BufferAppenderChunk.cs new file mode 100644 index 00000000..f68acd67 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BufferAppenderChunk.cs @@ -0,0 +1,28 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using SeqCli.Forwarder.Filesystem; + +namespace SeqCli.Forwarder.Storage; + +sealed record BufferAppenderChunk(StoreFileAppender Appender) : IDisposable +{ + public long WriteHead { get; set; } + + public void Dispose() + { + Appender.Dispose(); + } +} diff --git a/src/SeqCli/Forwarder/Storage/BufferReader.cs b/src/SeqCli/Forwarder/Storage/BufferReader.cs new file mode 100644 index 00000000..f2746b8c --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BufferReader.cs @@ -0,0 +1,325 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using SeqCli.Forwarder.Filesystem; + +namespace SeqCli.Forwarder.Storage; + +/// +/// The read-side of a buffer. +/// +public sealed class BufferReader +{ + readonly StoreDirectory _storeDirectory; + BufferReaderHead? _discardingHead; + BufferReaderHead? _readHead; + List _sortedChunks; + + BufferReader(StoreDirectory storeDirectory) + { + _sortedChunks = new List(); + _storeDirectory = storeDirectory; + _discardingHead = null; + _readHead = null; + } + + public static BufferReader Open(StoreDirectory storeDirectory) + { + var reader = new BufferReader(storeDirectory); + reader.ReadChunks(); + + return reader; + } + + /// + /// Try fill a batch from the underlying file set. + /// + /// This method does not throw. + /// + /// This method is expected to be called in a loop to continue filling and processing batches as they're written. + /// + /// Once the batch is processed, call to advance the reader past it. + /// + /// The maximum size in bytes of a batch to read. If a single value between newlines is larger + /// than this size then it will be discarded rather than read. + /// The newline-delimited batch of values read. + /// True if a batch was filled. If this method returns false, then it means there is either no new + /// data to read, some oversize data was discarded, or an IO error was encountered. + public bool TryFillBatch(int maxSize, [NotNullWhen(true)] out BufferReaderBatch? batch) + { + /* + This is where the meat of the buffer reader lives. Reading batches runs in two broad steps: + + 1. If a previous batch overflowed the buffer then we're in "discard mode". + Scan through the offending chunk until a newline delimiter is found. + 2. After discarding, attempt to fill a buffer with as much data as possible + from the underlying chunks. + */ + + if (_discardingHead != null) + { + var discardingRentedArray = ArrayPool.Shared.Rent(maxSize); + + // NOTE: We don't use `maxSize` here, because we're discarding these bytes + // so it doesn't matter what size the target array is + var discardingBatchBuffer = discardingRentedArray.AsSpan(); + + while (_discardingHead != null) + { + var chunk = _sortedChunks[0]; + + // If the chunk has changed (it may have been deleted externally) + // then stop discarding + if (chunk.Name.Id != _discardingHead.Value.Chunk) + { + _discardingHead = null; + + ArrayPool.Shared.Return(discardingRentedArray); + break; + } + + var chunkHead = Head(chunk); + + // Attempt to fill the buffer with data from the underlying chunk + if (!TryFillChunk(chunk, + chunkHead with { CommitHead = _discardingHead.Value.CommitHead }, + discardingBatchBuffer, + out var fill)) + { + // If attempting to read from the chunk fails then remove it and carry on + // This is also done below in the regular read-loop if reading fails + _sortedChunks.RemoveAt(0); + _discardingHead = null; + + ArrayPool.Shared.Return(discardingRentedArray); + break; + } + + // Scan forwards for the next newline + var firstNewlineIndex = discardingBatchBuffer[..fill.Value].IndexOf((byte)'\n'); + + // If a newline was found then advance the reader to it and stop discarding + if (firstNewlineIndex >= 0) fill = firstNewlineIndex + 1; + + _discardingHead = _discardingHead.Value with + { + CommitHead = _discardingHead.Value.CommitHead + fill.Value + }; + _readHead = _discardingHead; + + var isChunkFinished = _discardingHead.Value.CommitHead == chunkHead.WriteHead; + + // If the chunk is finished or a newline is found then stop discarding + if (firstNewlineIndex >= 0 || (isChunkFinished && _sortedChunks.Count > 1)) + { + _discardingHead = null; + + ArrayPool.Shared.Return(discardingRentedArray); + break; + } + + // If there's more data in the chunk to read then loop back through + if (!isChunkFinished) continue; + + // If the chunk is finished but a newline wasn't found then refresh + // our set of chunks and loop back through + ReadChunks(); + + ArrayPool.Shared.Return(discardingRentedArray); + batch = null; + return false; + } + } + + // Fill a buffer with newline-delimited values + + var rentedArray = ArrayPool.Shared.Rent(maxSize); + var batchBuffer = rentedArray.AsSpan()[..maxSize]; + var batchLength = 0; + + BufferReaderHead? batchHead = null; + var chunkIndex = 0; + + // Try fill the buffer with as much data as possible + // by walking over all chunks + while (chunkIndex < _sortedChunks.Count) + { + var chunk = _sortedChunks[chunkIndex]; + var chunkHead = Head(chunk); + + if (!TryFillChunk(chunk, chunkHead, batchBuffer[batchLength..], out var fill)) + { + // If we can't read from this chunk anymore then remove it and continue + _sortedChunks.RemoveAt(chunkIndex); + continue; + } + + var isBufferFull = batchLength + fill == maxSize; + var isChunkFinished = fill == chunkHead.WriteHead; + + // If either the buffer has been filled or we've reached the end of a chunk + // then scan to the last newline + if (isBufferFull || isChunkFinished) + { + // If the chunk is finished then we expect this to immediately find a trailing newline + // NOTE: `Span.LastIndexOf` and similar methods are vectorized + var lastNewlineIndex = batchBuffer[batchLength..(batchLength + fill.Value)].LastIndexOf((byte)'\n'); + if (lastNewlineIndex == -1) + { + // If this isn't the last chunk then discard the trailing data and move on + if (isChunkFinished && chunkIndex < _sortedChunks.Count) + { + chunkIndex += 1; + continue; + } + + // If this is the first chunk then we've hit an oversize payload + if (chunkIndex == 0) + { + _discardingHead = new BufferReaderHead(chunk.Name.Id, chunkHead.CommitHead + fill.Value); + + // Ensures we don't attempt to yield the data we've read + batchHead = null; + } + + // If the chunk isn't finished then the buffer is full + break; + } + + fill = lastNewlineIndex + 1; + } + + batchLength += fill.Value; + batchHead = new BufferReaderHead(chunk.Name.Id, chunkHead.CommitHead + fill.Value); + + chunkIndex += 1; + } + + // If the batch is empty (because there are no chunks or there's no new data) + // then refresh the set of chunks and return + if (batchHead == null || batchLength == 0) + { + ReadChunks(); + + ArrayPool.Shared.Return(rentedArray); + batch = null; + return false; + } + + // If the batch is non-empty then return it + batch = new BufferReaderBatch(batchHead.Value, ArrayPool.Shared, rentedArray, batchLength); + return true; + } + + /// + /// Advance the reader over a previously read batch. + /// + /// This method does not throw. + /// + /// The new head to resume reading from. + public void AdvanceTo(BufferReaderHead newReaderHead) + { + var removeLength = 0; + foreach (var chunk in _sortedChunks) + { + // A portion of the chunk is being skipped + if (chunk.Name.Id == newReaderHead.Chunk) break; + + // The remainder of the chunk is being skipped + if (chunk.Name.Id < newReaderHead.Chunk) + _storeDirectory.TryDelete(chunk.Name.ToString()); + else + throw new Exception("Chunks are out of order."); + + removeLength += 1; + } + + _readHead = newReaderHead; + _sortedChunks.RemoveRange(0, removeLength); + } + + BufferReaderChunkHead Head(BufferReaderChunk chunk) + { + if (_readHead != null && chunk.Name.Id == _readHead.Value.Chunk) + return chunk.Chunk.TryGetLength(out var writeHead) + ? new BufferReaderChunkHead(Math.Min(_readHead.Value.CommitHead, writeHead.Value), writeHead.Value) + : new BufferReaderChunkHead(_readHead.Value.CommitHead, _readHead.Value.CommitHead); + + chunk.Chunk.TryGetLength(out var length); + return new BufferReaderChunkHead(0, length ?? 0); + } + + void ReadChunks() + { + var head = _readHead ?? new BufferReaderHead(0, 0); + + List chunks = new(); + + foreach (var (fileName, file) in _storeDirectory + .List(candidateName => Path.GetExtension(candidateName) is ".clef")) + { + if (!ChunkName.TryParse(fileName, out var parsedChunkName)) continue; + + if (parsedChunkName.Value.Id >= head.Chunk) + chunks.Add(new BufferReaderChunk(parsedChunkName.Value, file)); + else + // If the chunk is before the one we're expecting to read then delete it; we've already processed it + _storeDirectory.TryDelete(fileName); + } + + chunks.Sort((a, b) => a.Name.Id.CompareTo(b.Name.Id)); + + var toDispose = _sortedChunks; + _sortedChunks = chunks; + + foreach (var chunk in toDispose) + try + { + chunk.Dispose(); + } + catch + { + // Ignored + } + } + + static bool TryFillChunk(BufferReaderChunk chunk, BufferReaderChunkHead chunkHead, Span buffer, + [NotNullWhen(true)] out int? filled) + { + var remaining = buffer.Length; + var fill = (int)Math.Min(remaining, chunkHead.Unadvanced); + + try + { + if (!chunk.TryCopyTo(buffer, chunkHead, fill)) + { + filled = null; + return false; + } + + filled = fill; + return true; + } + catch (IOException) + { + filled = null; + return false; + } + } +} diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs b/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs new file mode 100644 index 00000000..069c7535 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs @@ -0,0 +1,50 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Buffers; + +namespace SeqCli.Forwarder.Storage; + +/// +/// A contiguous batch of records pulled from a reader. +/// +public readonly record struct BufferReaderBatch +{ + readonly int _length; + + readonly ArrayPool? _pool; + readonly byte[] _storage; + + public BufferReaderBatch(BufferReaderHead readerHead, ArrayPool? pool, byte[] storage, int length) + { + ReaderHead = readerHead; + + _pool = pool; + _storage = storage; + _length = length; + } + + public BufferReaderHead ReaderHead { get; } + + public ReadOnlySpan AsSpan() + { + return _storage.AsSpan()[.._length]; + } + + public void Return() + { + _pool?.Return(_storage); + } +} diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderChunk.cs b/src/SeqCli/Forwarder/Storage/BufferReaderChunk.cs new file mode 100644 index 00000000..cf7f9b0c --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BufferReaderChunk.cs @@ -0,0 +1,56 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using SeqCli.Forwarder.Filesystem; + +namespace SeqCli.Forwarder.Storage; + +/// +/// An active chunk in a . +/// +record BufferReaderChunk(ChunkName Name, StoreFile Chunk) : IDisposable +{ + (long, StoreFileReader)? _reader; + + public void Dispose() + { + _reader?.Item2.Dispose(); + } + + public bool TryCopyTo(Span buffer, BufferReaderChunkHead head, int fill) + { + var readEnd = head.CommitHead + fill; + + if (_reader != null) + if (_reader.Value.Item1 < readEnd) + { + var toDispose = _reader.Value.Item2; + _reader = null; + + toDispose.Dispose(); + } + + if (_reader == null) + { + if (!Chunk.TryOpenRead(head.WriteHead, out var reader)) return false; + + _reader = (head.WriteHead, reader); + } + + _reader.Value.Item2.CopyTo(buffer, head.CommitHead, fill); + + return true; + } +} diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderChunkHead.cs b/src/SeqCli/Forwarder/Storage/BufferReaderChunkHead.cs new file mode 100644 index 00000000..a1778342 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BufferReaderChunkHead.cs @@ -0,0 +1,23 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace SeqCli.Forwarder.Storage; + +/// +/// The current position in a . +/// +public readonly record struct BufferReaderChunkHead(long CommitHead, long WriteHead) +{ + public long Unadvanced => WriteHead - CommitHead; +} diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderHead.cs b/src/SeqCli/Forwarder/Storage/BufferReaderHead.cs new file mode 100644 index 00000000..0240fa51 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BufferReaderHead.cs @@ -0,0 +1,20 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace SeqCli.Forwarder.Storage; + +/// +/// A position in a . +/// +public readonly record struct BufferReaderHead(ulong Chunk, long CommitHead); diff --git a/src/SeqCli/Forwarder/Storage/ChunkName.cs b/src/SeqCli/Forwarder/Storage/ChunkName.cs new file mode 100644 index 00000000..fc301cf4 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/ChunkName.cs @@ -0,0 +1,56 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Diagnostics.CodeAnalysis; + +namespace SeqCli.Forwarder.Storage; + +/// +/// A chunk file name with its incrementing identifier. +/// +public readonly record struct ChunkName +{ + readonly string _name; + + public readonly ulong Id; + + public ChunkName(ulong id) + { + Id = id; + _name = Identifier.Format(id, ".clef"); + } + + ChunkName(ulong id, string name) + { + Id = id; + _name = name; + } + + public static bool TryParse(string name, [NotNullWhen(true)] out ChunkName? parsed) + { + if (Identifier.TryParse(name, ".clef", out var id)) + { + parsed = new ChunkName(id.Value, name); + return true; + } + + parsed = null; + return false; + } + + public override string ToString() + { + return _name; + } +} diff --git a/src/SeqCli/Forwarder/Storage/Identifier.cs b/src/SeqCli/Forwarder/Storage/Identifier.cs new file mode 100644 index 00000000..000e0f07 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/Identifier.cs @@ -0,0 +1,60 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; + +namespace SeqCli.Forwarder.Storage; + +/// +/// Utilities for parsing and formatting file names with sortable identifiers. +/// +public static class Identifier +{ + /// + /// Try parse the identifier from the given name with the given extension. + /// + public static bool TryParse(string name, string extension, [NotNullWhen(true)] out ulong? parsed) + { + if (name.Length != 16 + extension.Length) + { + parsed = null; + return false; + } + + if (!name.EndsWith(extension)) + { + parsed = null; + return false; + } + + if (ulong.TryParse(name.AsSpan()[..16], NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var id)) + { + parsed = id; + return true; + } + + parsed = null; + return false; + } + + /// + /// Format an identifier with the given identifier and extension. + /// + public static string Format(ulong id, string extension) + { + return $"{id:x16}{extension}"; + } +} diff --git a/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs b/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs deleted file mode 100644 index 4477b926..00000000 --- a/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs +++ /dev/null @@ -1,6 +0,0 @@ -using System; -using System.Threading.Tasks; - -namespace SeqCli.Forwarder.Storage; - -public readonly record struct LogBufferEntry(byte[] Storage, Range Range, TaskCompletionSource Completion); diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index e511336e..e98c3828 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -28,8 +28,8 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using SeqCli.Config; +using SeqCli.Forwarder.Channel; using SeqCli.Forwarder.Diagnostics; -using SeqCli.Forwarder.Storage; using JsonException = System.Text.Json.JsonException; using JsonSerializer = Newtonsoft.Json.JsonSerializer; @@ -40,17 +40,17 @@ class IngestionEndpoints : IMapEndpoints static readonly Encoding Utf8 = new UTF8Encoding(false); readonly ConnectionConfig _connectionConfig; - readonly LogBufferMap _logBuffers; + readonly LogChannelMap _logChannels; readonly JsonSerializer _rawSerializer = JsonSerializer.Create( new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); public IngestionEndpoints( SeqCliConfig config, - LogBufferMap logBuffers) + LogChannelMap logChannels) { _connectionConfig = config.Connection; - _logBuffers = logBuffers; + _logChannels = logChannels; } public void MapEndpoints(WebApplication app) @@ -105,7 +105,7 @@ async Task IngestCompactFormat(HttpContext context) var cts = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted); cts.CancelAfter(TimeSpan.FromSeconds(5)); - var log = _logBuffers.Get(ApiKey(context.Request)); + var log = _logChannels.Get(ApiKey(context.Request)); var payload = ArrayPool.Shared.Rent(1024 * 1024 * 10); var writeHead = 0; @@ -235,7 +235,7 @@ static bool ValidateClef(Span evt) return true; } - static async Task Write(LogBuffer log, ArrayPool pool, byte[] storage, Range range, CancellationToken cancellationToken) + static async Task Write(LogChannel log, ArrayPool pool, byte[] storage, Range range, CancellationToken cancellationToken) { try { diff --git a/src/SeqCli/Forwarder/Web/Host/ServerService.cs b/src/SeqCli/Forwarder/Web/Host/ServerService.cs index ee4b473b..6832e953 100644 --- a/src/SeqCli/Forwarder/Web/Host/ServerService.cs +++ b/src/SeqCli/Forwarder/Web/Host/ServerService.cs @@ -15,8 +15,8 @@ using System; using System.Threading.Tasks; using Microsoft.Extensions.Hosting; +using SeqCli.Forwarder.Channel; using SeqCli.Forwarder.Diagnostics; -using SeqCli.Forwarder.Storage; using Serilog; namespace SeqCli.Forwarder.Web.Host; @@ -24,13 +24,13 @@ namespace SeqCli.Forwarder.Web.Host; class ServerService { readonly IHost _host; - readonly LogBufferMap _logBufferMap; + readonly LogChannelMap _logChannelMap; readonly string _listenUri; - public ServerService(IHost host, LogBufferMap logBufferMap, string listenUri) + public ServerService(IHost host, LogChannelMap logChannelMap, string listenUri) { _host = host; - _logBufferMap = logBufferMap; + _logChannelMap = logChannelMap; _listenUri = listenUri; } @@ -60,6 +60,6 @@ public async Task StopAsync() Log.Information("HTTP server stopped; flushing buffers..."); - await _logBufferMap.StopAsync(); + await _logChannelMap.StopAsync(); } } \ No newline at end of file diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index fbb39cf1..46b724a3 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -18,6 +18,7 @@ true true true + true WINDOWS diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreDirectory.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreDirectory.cs new file mode 100644 index 00000000..7382bc1d --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreDirectory.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using SeqCli.Forwarder.Filesystem; + +namespace SeqCli.Tests.Forwarder.Filesystem; + +public class InMemoryStoreDirectory : StoreDirectory +{ + readonly Dictionary _files = new(); + + public IReadOnlyDictionary Files => _files; + + public override InMemoryStoreFile Create(string name) + { + if (_files.ContainsKey(name)) throw new Exception($"The file {name} already exists"); + + _files.Add(name, new InMemoryStoreFile()); + + return _files[name]; + } + + public InMemoryStoreFile Create(string name, Span contents) + { + var file = Create(name); + file.Append(contents); + + return file; + } + + public override bool TryDelete(string name) + { + return _files.Remove(name); + } + + public override InMemoryStoreFile Replace(string toReplace, string replaceWith) + { + _files[toReplace] = _files[replaceWith]; + _files.Remove(replaceWith); + + return _files[toReplace]; + } + + public override IEnumerable<(string Name, StoreFile File)> List(Func predicate) + { + return _files + .Where(kv => predicate(kv.Key)) + .Select(kv => (kv.Key, kv.Value as StoreFile)); + } +} diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs new file mode 100644 index 00000000..1a48b246 --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs @@ -0,0 +1,38 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using SeqCli.Forwarder.Filesystem; + +namespace SeqCli.Tests.Forwarder.Filesystem; + +public class InMemoryStoreFile : StoreFile +{ + public byte[] Contents { get; private set; } = Array.Empty(); + + public override bool TryGetLength([NotNullWhen(true)] out long? length) + { + length = Contents.Length; + return true; + } + + public void Append(Span incoming) + { + var newContents = new byte[Contents.Length + incoming.Length]; + + Contents.CopyTo(newContents.AsSpan()); + incoming.CopyTo(newContents.AsSpan()[^incoming.Length..]); + + Contents = newContents; + } + + public override bool TryOpenRead(long length, [NotNullWhen(true)] out StoreFileReader? reader) + { + reader = new InMemoryStoreFileReader(this, (int)length); + return true; + } + + public override bool TryOpenAppend([NotNullWhen(true)] out StoreFileAppender? appender) + { + appender = new InMemoryStoreFileAppender(this); + return true; + } +} diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileAppender.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileAppender.cs new file mode 100644 index 00000000..aeaf66e1 --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileAppender.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using SeqCli.Forwarder.Filesystem; + +namespace SeqCli.Tests.Forwarder.Filesystem; + +public class InMemoryStoreFileAppender : StoreFileAppender +{ + readonly List _incoming; + + readonly InMemoryStoreFile _storeFile; + + public InMemoryStoreFileAppender(InMemoryStoreFile storeFile) + { + _storeFile = storeFile; + _incoming = new List(); + } + + public override void Append(Span data) + { + _incoming.AddRange(data); + } + + public override long Commit() + { + _storeFile.Append(_incoming.ToArray()); + _incoming.Clear(); + + return _storeFile.Contents.Length; + } + + public override void Sync() + { + } + + public override void Dispose() + { + } +} diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileReader.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileReader.cs new file mode 100644 index 00000000..56bd0838 --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileReader.cs @@ -0,0 +1,29 @@ +using System; +using SeqCli.Forwarder.Filesystem; + +namespace SeqCli.Tests.Forwarder.Filesystem; + +public class InMemoryStoreFileReader : StoreFileReader +{ + readonly int _length; + + readonly InMemoryStoreFile _storeFile; + + public InMemoryStoreFileReader(InMemoryStoreFile storeFile, int length) + { + _storeFile = storeFile; + _length = length; + } + + public override long CopyTo(Span buffer, long from = 0, long? length = null) + { + var span = _storeFile.Contents.AsSpan().Slice((int)from, (int?)length ?? _length); + span.CopyTo(buffer); + + return span.Length; + } + + public override void Dispose() + { + } +} diff --git a/test/SeqCli.Tests/Forwarder/Storage/BookmarkTests.cs b/test/SeqCli.Tests/Forwarder/Storage/BookmarkTests.cs new file mode 100644 index 00000000..3d67ecbd --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Storage/BookmarkTests.cs @@ -0,0 +1,83 @@ +using System.Linq; +using SeqCli.Forwarder.Storage; +using SeqCli.Tests.Forwarder.Filesystem; +using Xunit; + +namespace SeqCli.Tests.Forwarder.Storage; + +public class BookmarkTests +{ + [Fact] + public void CreateSetGet() + { + var directory = new InMemoryStoreDirectory(); + + var bookmark = Bookmark.Open(directory); + + Assert.False(bookmark.TryGet(out var value)); + Assert.Null(value); + + Assert.True(bookmark.TrySet(new BookmarkValue(42, 1))); + Assert.True(bookmark.TryGet(out value)); + Assert.Equal(new BookmarkValue(42, 1), value.Value); + + Assert.True(bookmark.TrySet(new BookmarkValue(42, int.MaxValue))); + Assert.True(bookmark.TryGet(out value)); + Assert.Equal(new BookmarkValue(42, int.MaxValue), value.Value); + } + + [Fact] + public void OpenDeletesOldBookmarks() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create($"{1L:x16}.bookmark", new BookmarkValue(3, 3478).Encode()); + directory.Create($"{3L:x16}.bookmark", new BookmarkValue(42, 17).Encode()); + + Assert.Equal(2, directory.Files.Count); + + var bookmark = Bookmark.Open(directory); + + Assert.Equal($"{3L:x16}.bookmark", directory.Files.Single().Key); + + Assert.True(bookmark.TryGet(out var value)); + Assert.Equal(new BookmarkValue(42, 17), value); + } + + [Fact] + public void OpenDeletesCorruptedBookmarks() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create($"{1L:x16}.bookmark", new BookmarkValue(3, 3478).Encode()); + + // This bookmark is invalid + directory.Create($"{3L:x16}.bookmark", new byte[] { 1, 2, 3 }); + + var bookmark = Bookmark.Open(directory); + + Assert.Empty(directory.Files); + + Assert.True(bookmark.TrySet(new BookmarkValue(42, 17))); + + Assert.Equal($"{4L:x16}.bookmark", directory.Files.Single().Key); + } + + [Fact] + public void OpenDeletesMisnamedBookmarks() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create($"{1L:x16}.bookmark", new BookmarkValue(3, 3478).Encode()); + + // This bookmark is invalid + directory.Create($"ff{3L:x16}.bookmark", new BookmarkValue(42, 17).Encode()); + + var bookmark = Bookmark.Open(directory); + + Assert.Single(directory.Files); + + Assert.True(bookmark.TryGet(out var value)); + Assert.Equal(new BookmarkValue(3, 3478), value); + } +} diff --git a/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs b/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs new file mode 100644 index 00000000..a217615b --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs @@ -0,0 +1,283 @@ +using System.Linq; +using SeqCli.Forwarder.Storage; +using SeqCli.Tests.Forwarder.Filesystem; +using Xunit; + +namespace SeqCli.Tests.Forwarder.Storage; + +public class BufferTests +{ + [Fact] + public void OpenAppendRead() + { + var directory = new InMemoryStoreDirectory(); + + using var writer = BufferAppender.Open(directory); + var reader = BufferReader.Open(directory); + + Assert.Equal(0, directory.Files.Count); + + // Append a payload + Assert.True(writer.TryAppend("{\"id\":1}\n"u8.ToArray(), long.MaxValue)); + Assert.Equal(1, directory.Files.Count); + + // Read the payload + Assert.False(reader.TryFillBatch(10, out _)); + Assert.True(reader.TryFillBatch(10, out var batch)); + var batchBuffer = batch.Value; + Assert.Equal(new BufferReaderHead(1, 9), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":1}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + + // Advance the reader + reader.AdvanceTo(batchBuffer.ReaderHead); + Assert.False(reader.TryFillBatch(10, out batch)); + + // Append another payload + Assert.True(writer.TryAppend("{\"id\":2}\n"u8.ToArray(), long.MaxValue)); + Assert.Equal(1, directory.Files.Count); + + // Read the payload + Assert.True(reader.TryFillBatch(10, out batch)); + batchBuffer = batch.Value; + Assert.Equal(new BufferReaderHead(1, 18), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":2}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + + // Advance the reader + reader.AdvanceTo(batchBuffer.ReaderHead); + Assert.False(reader.TryFillBatch(10, out batch)); + } + + [Fact] + public void ReadWaitsUntilCompleteDataOnLastChunk() + { + var directory = new InMemoryStoreDirectory(); + + var chunk = directory.Create(new ChunkName(1).ToString(), "{\"id\":1"u8.ToArray()); + + var reader = BufferReader.Open(directory); + + Assert.False(reader.TryFillBatch(512, out _)); + + chunk.Append("}"u8.ToArray()); + + Assert.False(reader.TryFillBatch(512, out _)); + + chunk.Append("\n"u8.ToArray()); + + Assert.True(reader.TryFillBatch(512, out var batch)); + var batchBuffer = batch.Value; + + Assert.Equal(new BufferReaderHead(1, 9), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":1}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + } + + [Fact] + public void ReadDiscardsPreviouslyReadChunks() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create(new ChunkName(1).ToString(), "{\"id\":1}\n"u8.ToArray()); + directory.Create(new ChunkName(2).ToString(), "{\"id\":2}\n"u8.ToArray()); + + var reader = BufferReader.Open(directory); + + Assert.True(reader.TryFillBatch(512, out var batch)); + var batchBuffer = batch.Value; + Assert.Equal(new BufferReaderHead(2, 9), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":1}\n{\"id\":2}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + + Assert.Equal(2, directory.Files.Count); + + reader.AdvanceTo(batchBuffer.ReaderHead); + + Assert.Equal(1, directory.Files.Count); + + directory.Create(new ChunkName(1).ToString(), "{\"id\":1}\n"u8.ToArray()); + directory.Create(new ChunkName(3).ToString(), "{\"id\":3}\n"u8.ToArray()); + + Assert.Equal(3, directory.Files.Count); + + Assert.False(reader.TryFillBatch(512, out _)); + Assert.True(reader.TryFillBatch(512, out batch)); + batchBuffer = batch.Value; + Assert.Equal(new BufferReaderHead(3, 9), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":3}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + + reader.AdvanceTo(batchBuffer.ReaderHead); + + Assert.Equal(1, directory.Files.Count); + } + + [Fact] + public void ReadDiscardsOversizePayloads() + { + var directory = new InMemoryStoreDirectory(); + + using var writer = BufferAppender.Open(directory); + + Assert.True(writer.TryAppend("{\"id\":1}\n"u8.ToArray(), long.MaxValue)); + + var reader = BufferReader.Open(directory); + + Assert.False(reader.TryFillBatch(5, out _)); + Assert.False(reader.TryFillBatch(512, out _)); + } + + [Fact] + public void ReadDoesNotDiscardAcrossFiles() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create(new ChunkName(1).ToString(), "{\"id\":1}"u8.ToArray()); + directory.Create(new ChunkName(2).ToString(), "{\"id\":2}\n"u8.ToArray()); + + var reader = BufferReader.Open(directory); + + Assert.True(reader.TryFillBatch(512, out var batch)); + var batchBuffer = batch.Value; + + Assert.Equal(new BufferReaderHead(2, 9), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":2}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + } + + [Fact] + public void ReadStopsDiscardingOnExternalDelete() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create(new ChunkName(1).ToString(), "{\"id\":1}"u8.ToArray()); + + var reader = BufferReader.Open(directory); + + Assert.False(reader.TryFillBatch(5, out _)); + + // Deleting the file here will cause our discarding chunk to change + Assert.True(directory.TryDelete(new ChunkName(1).ToString())); + directory.Create(new ChunkName(2).ToString(), "{\"id\":2}\n"u8.ToArray()); + + Assert.False(reader.TryFillBatch(512, out _)); + Assert.True(reader.TryFillBatch(512, out var batch)); + var batchBuffer = batch.Value; + + Assert.Equal(new BufferReaderHead(2, 9), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":2}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + } + + [Fact] + public void ReadStopsDiscardingOnExternalCreate() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create(new ChunkName(1).ToString(), "{\"id\":1}"u8.ToArray()); + + var reader = BufferReader.Open(directory); + + Assert.False(reader.TryFillBatch(5, out _)); + + // Creating a new file here will cause a new one to be created + directory.Create(new ChunkName(2).ToString(), "{\"id\":2}\n"u8.ToArray()); + + Assert.False(reader.TryFillBatch(512, out _)); + Assert.True(reader.TryFillBatch(512, out var batch)); + var batchBuffer = batch.Value; + + Assert.Equal(new BufferReaderHead(2, 9), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":2}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + } + + [Fact] + public void AppendRolloverOnWrite() + { + var directory = new InMemoryStoreDirectory(); + + using var writer = BufferAppender.Open(directory); + var reader = BufferReader.Open(directory); + + Assert.Equal(0, directory.Files.Count); + + Assert.True(writer.TryAppend("{\"id\":1}\n"u8.ToArray(), 17)); + Assert.True(writer.TryAppend("{\"id\":2}\n"u8.ToArray(), 17)); + + Assert.Equal(1, directory.Files.Count); + + Assert.True(writer.TryAppend("{\"id\":3}\n"u8.ToArray(), 17)); + + Assert.Equal(2, directory.Files.Count); + + Assert.False(reader.TryFillBatch(512, out _)); + Assert.True(reader.TryFillBatch(512, out var batch)); + var batchBuffer = batch.Value; + + Assert.Equal(new BufferReaderHead(2, 9), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":1}\n{\"id\":2}\n{\"id\":3}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + + reader.AdvanceTo(batchBuffer.ReaderHead); + + Assert.Equal(1, directory.Files.Count); + } + + [Fact] + public void ExistingFilesAreReadonly() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create(new ChunkName(0).ToString()); + + using var writer = BufferAppender.Open(directory); + var reader = BufferReader.Open(directory); + + Assert.Equal(1, directory.Files.Count); + + Assert.True(writer.TryAppend("{\"id\":1}\n"u8.ToArray(), long.MaxValue)); + + Assert.Equal(2, directory.Files.Count); + + Assert.False(reader.TryFillBatch(512, out _)); + Assert.True(reader.TryFillBatch(512, out var batch)); + var batchBuffer = batch.Value; + + Assert.Equal(new BufferReaderHead(1, 9), batchBuffer.ReaderHead); + } + + [Fact] + public void OpenReadAcrossChunks() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create(new ChunkName(0).ToString(), "{\"id\":1}\n"u8.ToArray()); + directory.Create(new ChunkName(1).ToString(), "{\"id\":2}\n"u8.ToArray()); + directory.Create(new ChunkName(2).ToString(), "{\"id\":3}\n"u8.ToArray()); + + var reader = BufferReader.Open(directory); + + Assert.Equal(3, directory.Files.Count); + + Assert.True(reader.TryFillBatch(512, out var batch)); + var batchBuffer = batch.Value; + + Assert.Equal("{\"id\":1}\n{\"id\":2}\n{\"id\":3}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + + reader.AdvanceTo(batchBuffer.ReaderHead); + + Assert.Single(directory.Files); + } + + [Fact] + public void MaxChunksOnAppender() + { + var directory = new InMemoryStoreDirectory(); + + using var appender = BufferAppender.Open(directory); + + for (var i = 0; i < 10; i++) Assert.True(appender.TryAppend("{\"id\":1}\n"u8.ToArray(), 5, 3)); + + var files = directory.Files.Select(f => f.Key).ToList(); + files.Sort(); + + Assert.Equal([ + "0000000000000008.clef", + "0000000000000009.clef", + "000000000000000a.clef" + ], files); + } +} diff --git a/test/SeqCli.Tests/Forwarder/Storage/IdentifierTests.cs b/test/SeqCli.Tests/Forwarder/Storage/IdentifierTests.cs new file mode 100644 index 00000000..b5d1f752 --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Storage/IdentifierTests.cs @@ -0,0 +1,31 @@ +using SeqCli.Forwarder.Storage; +using Xunit; + +namespace SeqCli.Tests.Forwarder.Storage; + +public class IdentifierTests +{ + [Theory] + [InlineData("0000000000000000.clef", 0)] + [InlineData("0000000000000001.clef", 1)] + [InlineData("000000000000000a.clef", 10)] + [InlineData("ffffffffffffffff.clef", ulong.MaxValue)] + public void ParseValid(string name, ulong expected) + { + Assert.True(ChunkName.TryParse(name, out var actual)); + + Assert.Equal(expected, actual.Value.Id); + Assert.Equal(name, actual.Value.ToString()); + } + + [Theory] + [InlineData("0.clef")] + [InlineData("one.clef")] + [InlineData("00000000000.clef.value")] + [InlineData("0ffffffffffffffff.clef")] + [InlineData("0xffffffffffffff.clef")] + public void ParseInvalid(string name) + { + Assert.False(ChunkName.TryParse(name, out _)); + } +} diff --git a/test/SeqCli.Tests/SeqCli.Tests.csproj b/test/SeqCli.Tests/SeqCli.Tests.csproj index 9ebc2e63..9f75b8f4 100644 --- a/test/SeqCli.Tests/SeqCli.Tests.csproj +++ b/test/SeqCli.Tests/SeqCli.Tests.csproj @@ -15,6 +15,7 @@ + From dc116bcf5e6d8b63788c78dca3841c84ce474b2b Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 9 May 2024 09:03:24 +1000 Subject: [PATCH 32/51] Fix tests --- src/SeqCli/Cli/Commands/ConfigCommand.cs | 13 +++++++-- .../Cli/Commands/Forwarder/RunCommand.cs | 19 ------------ .../Web/RequestProcessingException.cs | 29 ------------------- .../ProfileCreateListRemoveTestCase.cs | 4 +-- 4 files changed, 12 insertions(+), 53 deletions(-) delete mode 100644 src/SeqCli/Forwarder/Web/RequestProcessingException.cs diff --git a/src/SeqCli/Cli/Commands/ConfigCommand.cs b/src/SeqCli/Cli/Commands/ConfigCommand.cs index f991dc44..3cc565e6 100644 --- a/src/SeqCli/Cli/Commands/ConfigCommand.cs +++ b/src/SeqCli/Cli/Commands/ConfigCommand.cs @@ -19,6 +19,7 @@ using System.Linq; using System.Reflection; using System.Threading.Tasks; +using Newtonsoft.Json; using SeqCli.Config; using SeqCli.Util; using Serilog; @@ -142,11 +143,12 @@ static void List(SeqCliConfig config) static IEnumerable> ReadPairs(object config) { foreach (var property in config.GetType().GetTypeInfo().DeclaredProperties - .Where(p => p.CanRead && p.GetMethod!.IsPublic && !p.GetMethod.IsStatic && !p.Name.StartsWith("Encoded")) + .Select(p => new { Property = p, Name = GetConfigPropertyName(p)}) + .Where(p => p.Property.CanRead && p.Property.GetMethod!.IsPublic && !p.Property.GetMethod.IsStatic && !p.Name.StartsWith("encoded", StringComparison.OrdinalIgnoreCase)) .OrderBy(p => p.Name)) { var propertyName = Camelize(property.Name); - var propertyValue = property.GetValue(config); + var propertyValue = property.Property.GetValue(config); if (propertyValue is IDictionary dict) { @@ -175,11 +177,16 @@ static void List(SeqCliConfig config) } } + static string GetConfigPropertyName(PropertyInfo property) + { + return property.GetCustomAttribute()?.PropertyName ?? property.Name; + } + static string Camelize(string s) { if (s.Length < 2) throw new NotSupportedException("No camel-case support for short names"); - return char.ToLowerInvariant(s[0]) + s.Substring(1); + return char.ToLowerInvariant(s[0]) + s[1..]; } static string Format(object? value) diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index 4e69cebd..06f007fb 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -151,25 +151,6 @@ protected override async Task Run(string[] unrecognized) if (container == null) throw new Exception("Host did not build container."); - app.Use(async (context, next) => - { - try - { - await next(); - } - // ISSUE: this exception type isn't currently used. - catch (RequestProcessingException rex) - { - if (context.Response.HasStarted) - throw; - - context.Response.StatusCode = (int)rex.StatusCode; - context.Response.ContentType = "text/plain; charset=UTF-8"; - await context.Response.Body.WriteAsync(Encoding.UTF8.GetBytes(rex.Message)); - await context.Response.CompleteAsync(); - } - }); - foreach (var mapper in container.Resolve>()) { mapper.MapEndpoints(app); diff --git a/src/SeqCli/Forwarder/Web/RequestProcessingException.cs b/src/SeqCli/Forwarder/Web/RequestProcessingException.cs deleted file mode 100644 index bfef07af..00000000 --- a/src/SeqCli/Forwarder/Web/RequestProcessingException.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright © Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Net; - -namespace SeqCli.Forwarder.Web; - -class RequestProcessingException : Exception -{ - public RequestProcessingException(string message, HttpStatusCode statusCode = HttpStatusCode.BadRequest) - : base(message) - { - StatusCode = statusCode; - } - - public HttpStatusCode StatusCode { get; } -} \ No newline at end of file diff --git a/test/SeqCli.EndToEnd/Profile/ProfileCreateListRemoveTestCase.cs b/test/SeqCli.EndToEnd/Profile/ProfileCreateListRemoveTestCase.cs index 17661e59..9adc50ed 100644 --- a/test/SeqCli.EndToEnd/Profile/ProfileCreateListRemoveTestCase.cs +++ b/test/SeqCli.EndToEnd/Profile/ProfileCreateListRemoveTestCase.cs @@ -14,13 +14,13 @@ public Task ExecuteAsync(SeqConnection connection, ILogger logger, CliCommandRun Assert.Equal(0, create); Assert.Equal(0, runner.Exec("profile list", disconnected: true)); - Assert.Contains("test (https://seq.example.com)", runner.LastRunProcess.Output); + Assert.Contains("test (https://seq.example.com)", runner.LastRunProcess!.Output); Assert.Equal(0, runner.Exec("config", disconnected: true)); Assert.Contains("profiles[test].serverUrl", runner.LastRunProcess.Output); Assert.Contains("https://seq.example.com", runner.LastRunProcess.Output); Assert.Contains("profiles[test].apiKey", runner.LastRunProcess.Output); - Assert.Contains("123", runner.LastRunProcess.Output); + Assert.Contains("pd.", runner.LastRunProcess.Output); var remove = runner.Exec("profile remove", "-n Test", disconnected: true); Assert.Equal(0, remove); From aeb3b5f68bb4b4d054f6856edf80f380909a11ce Mon Sep 17 00:00:00 2001 From: KodrAus Date: Thu, 9 May 2024 09:13:48 +1000 Subject: [PATCH 33/51] fix up warnings in tests --- .../Forwarder/Filesystem/InMemoryStoreFile.cs | 2 ++ .../Forwarder/Storage/BufferTests.cs | 18 +++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs index 1a48b246..dd2b54e8 100644 --- a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs @@ -1,3 +1,5 @@ +#nullable enable + using System; using System.Diagnostics.CodeAnalysis; using SeqCli.Forwarder.Filesystem; diff --git a/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs b/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs index a217615b..86129275 100644 --- a/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs +++ b/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs @@ -15,11 +15,11 @@ public void OpenAppendRead() using var writer = BufferAppender.Open(directory); var reader = BufferReader.Open(directory); - Assert.Equal(0, directory.Files.Count); + Assert.Empty(directory.Files); // Append a payload Assert.True(writer.TryAppend("{\"id\":1}\n"u8.ToArray(), long.MaxValue)); - Assert.Equal(1, directory.Files.Count); + Assert.Single(directory.Files); // Read the payload Assert.False(reader.TryFillBatch(10, out _)); @@ -34,7 +34,7 @@ public void OpenAppendRead() // Append another payload Assert.True(writer.TryAppend("{\"id\":2}\n"u8.ToArray(), long.MaxValue)); - Assert.Equal(1, directory.Files.Count); + Assert.Single(directory.Files); // Read the payload Assert.True(reader.TryFillBatch(10, out batch)); @@ -90,7 +90,7 @@ public void ReadDiscardsPreviouslyReadChunks() reader.AdvanceTo(batchBuffer.ReaderHead); - Assert.Equal(1, directory.Files.Count); + Assert.Single(directory.Files); directory.Create(new ChunkName(1).ToString(), "{\"id\":1}\n"u8.ToArray()); directory.Create(new ChunkName(3).ToString(), "{\"id\":3}\n"u8.ToArray()); @@ -105,7 +105,7 @@ public void ReadDiscardsPreviouslyReadChunks() reader.AdvanceTo(batchBuffer.ReaderHead); - Assert.Equal(1, directory.Files.Count); + Assert.Single(directory.Files); } [Fact] @@ -193,12 +193,12 @@ public void AppendRolloverOnWrite() using var writer = BufferAppender.Open(directory); var reader = BufferReader.Open(directory); - Assert.Equal(0, directory.Files.Count); + Assert.Empty(directory.Files); Assert.True(writer.TryAppend("{\"id\":1}\n"u8.ToArray(), 17)); Assert.True(writer.TryAppend("{\"id\":2}\n"u8.ToArray(), 17)); - Assert.Equal(1, directory.Files.Count); + Assert.Single(directory.Files); Assert.True(writer.TryAppend("{\"id\":3}\n"u8.ToArray(), 17)); @@ -213,7 +213,7 @@ public void AppendRolloverOnWrite() reader.AdvanceTo(batchBuffer.ReaderHead); - Assert.Equal(1, directory.Files.Count); + Assert.Single(directory.Files); } [Fact] @@ -226,7 +226,7 @@ public void ExistingFilesAreReadonly() using var writer = BufferAppender.Open(directory); var reader = BufferReader.Open(directory); - Assert.Equal(1, directory.Files.Count); + Assert.Single(directory.Files); Assert.True(writer.TryAppend("{\"id\":1}\n"u8.ToArray(), long.MaxValue)); From 6ce99a46d35ac02fb846d58ba6954b10728b6bf0 Mon Sep 17 00:00:00 2001 From: KodrAus Date: Thu, 9 May 2024 09:29:35 +1000 Subject: [PATCH 34/51] remove extra csproj item --- test/SeqCli.Tests/SeqCli.Tests.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/test/SeqCli.Tests/SeqCli.Tests.csproj b/test/SeqCli.Tests/SeqCli.Tests.csproj index e46c0ee0..1bf867fa 100644 --- a/test/SeqCli.Tests/SeqCli.Tests.csproj +++ b/test/SeqCli.Tests/SeqCli.Tests.csproj @@ -15,7 +15,6 @@ - From cc8f67a942791645cb81001302e82418a317a201 Mon Sep 17 00:00:00 2001 From: KodrAus Date: Thu, 9 May 2024 10:19:49 +1000 Subject: [PATCH 35/51] fix up windows build --- Build.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Build.ps1 b/Build.ps1 index 29f5ea0f..efd19536 100644 --- a/Build.ps1 +++ b/Build.ps1 @@ -32,7 +32,7 @@ function Create-ArtifactDir function Publish-Archives($version) { - $rids = $([xml](Get-Content .\src\SeqCli\SeqCli.csproj)).Project.PropertyGroup.RuntimeIdentifiers.Split(';') + $rids = $([xml](Get-Content .\src\SeqCli\SeqCli.csproj)).Project.PropertyGroup.RuntimeIdentifiers[0].Split(';') foreach ($rid in $rids) { $tfm = $framework if ($rid -eq "win-x64") { From a8ba8430a88cb0919ecc4bffc3a942729116d7e0 Mon Sep 17 00:00:00 2001 From: KodrAus Date: Thu, 9 May 2024 15:29:33 +1000 Subject: [PATCH 36/51] fix up tfms --- src/SeqCli/SeqCli.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index 61f7f087..c15f93fb 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -1,7 +1,7 @@  Exe - net8.0 + net8.0;net8.0-windows seqcli ..\..\asset\SeqCli.ico win-x64;linux-x64;linux-musl-x64;osx-x64;linux-arm64;linux-musl-arm64;osx-arm64 From b8b6171d68b60368b7082e00eda9f5d606901bfc Mon Sep 17 00:00:00 2001 From: Ashley Mannix Date: Fri, 10 May 2024 14:27:21 +1000 Subject: [PATCH 37/51] updates based on PR review --- src/SeqCli/Apps/Hosting/AppContainer.cs | 2 +- src/SeqCli/Cli/Commands/App/InstallCommand.cs | 2 +- src/SeqCli/Cli/Commands/App/UpdateCommand.cs | 2 +- .../Commands/Bench/BenchCasesCollection.cs | 2 +- src/SeqCli/Cli/Commands/Bench/BenchCommand.cs | 2 +- .../Cli/Commands/Bench/QueryBenchCase.cs | 2 +- .../Commands/Bench/QueryBenchCaseTimings.cs | 2 +- .../Cli/Commands/Forwarder/StartCommand.cs | 2 +- .../Cli/Commands/Forwarder/StopCommand.cs | 2 +- .../Commands/Forwarder/UninstallCommand.cs | 2 +- src/SeqCli/Cli/Commands/Node/DemoteCommand.cs | 2 +- src/SeqCli/Cli/Commands/Node/HealthCommand.cs | 2 +- src/SeqCli/Cli/Commands/Node/ListCommand.cs | 2 +- .../Cli/Commands/Sample/IngestCommand.cs | 2 +- .../Cli/Commands/Sample/SetupCommand.cs | 2 +- .../Cli/Commands/Settings/ClearCommand.cs | 2 +- .../Cli/Commands/Settings/NamesCommand.cs | 2 +- .../Cli/Commands/Settings/SetCommand.cs | 2 +- .../Cli/Commands/Settings/ShowCommand.cs | 2 +- src/SeqCli/Cli/Features/TimeoutFeature.cs | 2 +- .../Filesystem/EmptyStoreFileReader.cs | 4 +-- .../Forwarder/Filesystem/StoreDirectory.cs | 4 +-- src/SeqCli/Forwarder/Filesystem/StoreFile.cs | 4 +-- .../Forwarder/Filesystem/StoreFileAppender.cs | 4 +-- .../Forwarder/Filesystem/StoreFileReader.cs | 4 +-- .../Filesystem/System/SystemStoreDirectory.cs | 11 +++++--- .../Filesystem/System/SystemStoreFile.cs | 4 +-- .../System/SystemStoreFileAppender.cs | 4 +-- .../System/SystemStoreFileReader.cs | 4 +-- .../Forwarder/Filesystem/System/Unix/Libc.cs | 6 +++-- src/SeqCli/Forwarder/ForwarderModule.cs | 2 +- src/SeqCli/Forwarder/Storage/Bookmark.cs | 4 +-- src/SeqCli/Forwarder/Storage/BookmarkName.cs | 4 +-- src/SeqCli/Forwarder/Storage/BookmarkValue.cs | 4 +-- .../Forwarder/Storage/BufferAppender.cs | 25 +++++++++++++------ .../Forwarder/Storage/BufferAppenderChunk.cs | 10 ++++++-- src/SeqCli/Forwarder/Storage/BufferReader.cs | 4 +-- .../Forwarder/Storage/BufferReaderBatch.cs | 4 +-- .../Forwarder/Storage/BufferReaderChunk.cs | 13 ++++++++-- .../Storage/BufferReaderChunkHead.cs | 4 +-- .../Forwarder/Storage/BufferReaderHead.cs | 4 +-- src/SeqCli/Forwarder/Storage/ChunkName.cs | 4 +-- src/SeqCli/Forwarder/Storage/Identifier.cs | 4 +-- .../Forwarder/Web/Api/ApiRootEndpoints.cs | 2 +- .../Forwarder/Web/Api/IngestionEndpoints.cs | 2 +- .../Web/Api/IngestionLogEndpoints.cs | 2 +- .../Forwarder/Web/Host/ServerService.cs | 2 +- src/SeqCli/Sample/Loader/Simulation.cs | 2 +- src/SeqCli/SeqCli.csproj | 3 +++ src/SeqCli/Templates/Ast/JsonTemplate.cs | 2 +- src/SeqCli/Templates/Ast/JsonTemplateArray.cs | 2 +- .../Templates/Ast/JsonTemplateBoolean.cs | 2 +- src/SeqCli/Templates/Ast/JsonTemplateCall.cs | 2 +- src/SeqCli/Templates/Ast/JsonTemplateNull.cs | 2 +- .../Templates/Ast/JsonTemplateNumber.cs | 2 +- .../Templates/Ast/JsonTemplateObject.cs | 2 +- .../Templates/Ast/JsonTemplateString.cs | 2 +- .../Evaluator/JsonTemplateEvaluator.cs | 2 +- .../Evaluator/JsonTemplateFunction.cs | 2 +- src/SeqCli/Templates/Import/EntityTemplate.cs | 2 +- .../Templates/Import/EntityTemplateLoader.cs | 2 +- src/SeqCli/Templates/Import/GenericEntity.cs | 2 +- .../Templates/Import/TemplateSetImporter.cs | 2 +- .../JsonTemplateObjectGraphConverter.cs | 2 +- .../Templates/Parser/JsonTemplateParser.cs | 2 +- .../Parser/JsonTemplateTextParsers.cs | 2 +- .../Templates/Parser/JsonTemplateToken.cs | 2 +- .../Templates/Parser/JsonTemplateTokenizer.cs | 2 +- src/SeqCli/Util/LogEventPropertyFactory.cs | 2 +- .../Filesystem/InMemoryStoreDirectory.cs | 4 +-- .../Forwarder/Filesystem/InMemoryStoreFile.cs | 2 +- .../Filesystem/InMemoryStoreFileAppender.cs | 2 +- .../Filesystem/InMemoryStoreFileReader.cs | 2 +- 73 files changed, 136 insertions(+), 102 deletions(-) diff --git a/src/SeqCli/Apps/Hosting/AppContainer.cs b/src/SeqCli/Apps/Hosting/AppContainer.cs index fe23a843..72184492 100644 --- a/src/SeqCli/Apps/Hosting/AppContainer.cs +++ b/src/SeqCli/Apps/Hosting/AppContainer.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/App/InstallCommand.cs b/src/SeqCli/Cli/Commands/App/InstallCommand.cs index 94d98482..7f9974d8 100644 --- a/src/SeqCli/Cli/Commands/App/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/App/InstallCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/App/UpdateCommand.cs b/src/SeqCli/Cli/Commands/App/UpdateCommand.cs index bf7fb40f..e0051a9c 100644 --- a/src/SeqCli/Cli/Commands/App/UpdateCommand.cs +++ b/src/SeqCli/Cli/Commands/App/UpdateCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Bench/BenchCasesCollection.cs b/src/SeqCli/Cli/Commands/Bench/BenchCasesCollection.cs index 441b6ae0..a47c8206 100644 --- a/src/SeqCli/Cli/Commands/Bench/BenchCasesCollection.cs +++ b/src/SeqCli/Cli/Commands/Bench/BenchCasesCollection.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Bench/BenchCommand.cs b/src/SeqCli/Cli/Commands/Bench/BenchCommand.cs index 8742d77b..0b538815 100644 --- a/src/SeqCli/Cli/Commands/Bench/BenchCommand.cs +++ b/src/SeqCli/Cli/Commands/Bench/BenchCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Bench/QueryBenchCase.cs b/src/SeqCli/Cli/Commands/Bench/QueryBenchCase.cs index ce31c26e..55bee6d2 100644 --- a/src/SeqCli/Cli/Commands/Bench/QueryBenchCase.cs +++ b/src/SeqCli/Cli/Commands/Bench/QueryBenchCase.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Bench/QueryBenchCaseTimings.cs b/src/SeqCli/Cli/Commands/Bench/QueryBenchCaseTimings.cs index a3bc8834..be8cd5ec 100644 --- a/src/SeqCli/Cli/Commands/Bench/QueryBenchCaseTimings.cs +++ b/src/SeqCli/Cli/Commands/Bench/QueryBenchCaseTimings.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs index 66f859fc..0f0e97d6 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs index 955c550c..14c9c9e7 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs index 50845d26..6805843e 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Node/DemoteCommand.cs b/src/SeqCli/Cli/Commands/Node/DemoteCommand.cs index 382d06a5..68358710 100644 --- a/src/SeqCli/Cli/Commands/Node/DemoteCommand.cs +++ b/src/SeqCli/Cli/Commands/Node/DemoteCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Node/HealthCommand.cs b/src/SeqCli/Cli/Commands/Node/HealthCommand.cs index 24dad8f1..0b82b63f 100644 --- a/src/SeqCli/Cli/Commands/Node/HealthCommand.cs +++ b/src/SeqCli/Cli/Commands/Node/HealthCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Node/ListCommand.cs b/src/SeqCli/Cli/Commands/Node/ListCommand.cs index 9b24b03c..1a69ee1a 100644 --- a/src/SeqCli/Cli/Commands/Node/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Node/ListCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Sample/IngestCommand.cs b/src/SeqCli/Cli/Commands/Sample/IngestCommand.cs index ff94f887..95644de5 100644 --- a/src/SeqCli/Cli/Commands/Sample/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/Sample/IngestCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Sample/SetupCommand.cs b/src/SeqCli/Cli/Commands/Sample/SetupCommand.cs index 8905c97c..71f5c952 100644 --- a/src/SeqCli/Cli/Commands/Sample/SetupCommand.cs +++ b/src/SeqCli/Cli/Commands/Sample/SetupCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Settings/ClearCommand.cs b/src/SeqCli/Cli/Commands/Settings/ClearCommand.cs index e4025de8..2a3ac781 100644 --- a/src/SeqCli/Cli/Commands/Settings/ClearCommand.cs +++ b/src/SeqCli/Cli/Commands/Settings/ClearCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Settings/NamesCommand.cs b/src/SeqCli/Cli/Commands/Settings/NamesCommand.cs index 40f060d5..d50345ab 100644 --- a/src/SeqCli/Cli/Commands/Settings/NamesCommand.cs +++ b/src/SeqCli/Cli/Commands/Settings/NamesCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Settings/SetCommand.cs b/src/SeqCli/Cli/Commands/Settings/SetCommand.cs index 67ad8844..8270f87e 100644 --- a/src/SeqCli/Cli/Commands/Settings/SetCommand.cs +++ b/src/SeqCli/Cli/Commands/Settings/SetCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Settings/ShowCommand.cs b/src/SeqCli/Cli/Commands/Settings/ShowCommand.cs index e351b41d..38582e80 100644 --- a/src/SeqCli/Cli/Commands/Settings/ShowCommand.cs +++ b/src/SeqCli/Cli/Commands/Settings/ShowCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Features/TimeoutFeature.cs b/src/SeqCli/Cli/Features/TimeoutFeature.cs index d5fb066b..3530812e 100644 --- a/src/SeqCli/Cli/Features/TimeoutFeature.cs +++ b/src/SeqCli/Cli/Features/TimeoutFeature.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Forwarder/Filesystem/EmptyStoreFileReader.cs b/src/SeqCli/Forwarder/Filesystem/EmptyStoreFileReader.cs index 0410e23a..bc8ac092 100644 --- a/src/SeqCli/Forwarder/Filesystem/EmptyStoreFileReader.cs +++ b/src/SeqCli/Forwarder/Filesystem/EmptyStoreFileReader.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ namespace SeqCli.Forwarder.Filesystem; -public sealed class EmptyStoreFileReader : StoreFileReader +sealed class EmptyStoreFileReader : StoreFileReader { public override void Dispose() { diff --git a/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs b/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs index e9503162..39008865 100644 --- a/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs +++ b/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ namespace SeqCli.Forwarder.Filesystem; /// /// A container of s and their names. /// -public abstract class StoreDirectory +abstract class StoreDirectory { /// /// Create a new file with the given name, linking it into the filesystem. diff --git a/src/SeqCli/Forwarder/Filesystem/StoreFile.cs b/src/SeqCli/Forwarder/Filesystem/StoreFile.cs index 9e998523..1beed35f 100644 --- a/src/SeqCli/Forwarder/Filesystem/StoreFile.cs +++ b/src/SeqCli/Forwarder/Filesystem/StoreFile.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ namespace SeqCli.Forwarder.Filesystem; -public abstract class StoreFile +abstract class StoreFile { /// /// Get the length of this file. diff --git a/src/SeqCli/Forwarder/Filesystem/StoreFileAppender.cs b/src/SeqCli/Forwarder/Filesystem/StoreFileAppender.cs index 8e04dd91..31f7af4d 100644 --- a/src/SeqCli/Forwarder/Filesystem/StoreFileAppender.cs +++ b/src/SeqCli/Forwarder/Filesystem/StoreFileAppender.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ namespace SeqCli.Forwarder.Filesystem; -public abstract class StoreFileAppender : IDisposable +abstract class StoreFileAppender : IDisposable { public abstract void Dispose(); diff --git a/src/SeqCli/Forwarder/Filesystem/StoreFileReader.cs b/src/SeqCli/Forwarder/Filesystem/StoreFileReader.cs index d98feaad..5679c914 100644 --- a/src/SeqCli/Forwarder/Filesystem/StoreFileReader.cs +++ b/src/SeqCli/Forwarder/Filesystem/StoreFileReader.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ namespace SeqCli.Forwarder.Filesystem; -public abstract class StoreFileReader : IDisposable +abstract class StoreFileReader : IDisposable { public abstract void Dispose(); diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs index be27c3d4..e436185f 100644 --- a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,11 +16,14 @@ using System.Collections.Generic; using System.IO; using System.Runtime.InteropServices; + +#if UNIX using SeqCli.Forwarder.Filesystem.System.Unix; +#endif namespace SeqCli.Forwarder.Filesystem.System; -public sealed class SystemStoreDirectory : StoreDirectory +sealed class SystemStoreDirectory : StoreDirectory { readonly string _directoryPath; @@ -110,8 +113,7 @@ public override StoreFile ReplaceContents(string name, Span contents, bool static void Dirsync(string directoryPath) { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return; - +#if UNIX var dir = Libc.open(directoryPath, 0); if (dir == -1) return; @@ -121,5 +123,6 @@ static void Dirsync(string directoryPath) Libc.fsync(dir); Libc.close(dir); #pragma warning restore CA1806 +#endif } } diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFile.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFile.cs index 361b6741..55b37375 100644 --- a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFile.cs +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFile.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ namespace SeqCli.Forwarder.Filesystem.System; -public sealed class SystemStoreFile : StoreFile +sealed class SystemStoreFile : StoreFile { static readonly FileStreamOptions AppendOptions = new() { diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileAppender.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileAppender.cs index b64b9d26..298ea8d2 100644 --- a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileAppender.cs +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileAppender.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ namespace SeqCli.Forwarder.Filesystem.System; -public sealed class SystemStoreFileAppender : StoreFileAppender +sealed class SystemStoreFileAppender : StoreFileAppender { readonly FileStream _file; long _initialLength; diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileReader.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileReader.cs index 23e95101..22108eb5 100644 --- a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileReader.cs +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileReader.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ namespace SeqCli.Forwarder.Filesystem.System; -public sealed class SystemStoreFileReader : StoreFileReader +sealed class SystemStoreFileReader : StoreFileReader { readonly MemoryMappedViewAccessor _accessor; readonly MemoryMappedFile _file; diff --git a/src/SeqCli/Forwarder/Filesystem/System/Unix/Libc.cs b/src/SeqCli/Forwarder/Filesystem/System/Unix/Libc.cs index c9edc557..4561a20a 100644 --- a/src/SeqCli/Forwarder/Filesystem/System/Unix/Libc.cs +++ b/src/SeqCli/Forwarder/Filesystem/System/Unix/Libc.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#if UNIX using System.Runtime.InteropServices; namespace SeqCli.Forwarder.Filesystem.System.Unix; @@ -26,4 +27,5 @@ static class Libc [DllImport("libc")] public static extern int fsync(int fd); -} \ No newline at end of file +} +#endif diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 03d24974..0cc9c954 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Forwarder/Storage/Bookmark.cs b/src/SeqCli/Forwarder/Storage/Bookmark.cs index f3ec3ea4..3f75fb8b 100644 --- a/src/SeqCli/Forwarder/Storage/Bookmark.cs +++ b/src/SeqCli/Forwarder/Storage/Bookmark.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ namespace SeqCli.Forwarder.Storage; /// /// A durable bookmark of progress processing buffers. /// -public sealed class Bookmark +sealed class Bookmark { readonly StoreDirectory _storeDirectory; diff --git a/src/SeqCli/Forwarder/Storage/BookmarkName.cs b/src/SeqCli/Forwarder/Storage/BookmarkName.cs index 76206472..e40a8d8f 100644 --- a/src/SeqCli/Forwarder/Storage/BookmarkName.cs +++ b/src/SeqCli/Forwarder/Storage/BookmarkName.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ namespace SeqCli.Forwarder.Storage; /// /// A bookmark file name with its incrementing identifier. /// -public readonly record struct BookmarkName +readonly record struct BookmarkName { readonly string _name; diff --git a/src/SeqCli/Forwarder/Storage/BookmarkValue.cs b/src/SeqCli/Forwarder/Storage/BookmarkValue.cs index 72716391..7d86f90b 100644 --- a/src/SeqCli/Forwarder/Storage/BookmarkValue.cs +++ b/src/SeqCli/Forwarder/Storage/BookmarkValue.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ namespace SeqCli.Forwarder.Storage; /// /// The in-memory value of a bookmark. /// -public readonly record struct BookmarkValue(ulong Id, long CommitHead) +readonly record struct BookmarkValue(ulong Id, long CommitHead) { public void EncodeTo(Span bookmark) { diff --git a/src/SeqCli/Forwarder/Storage/BufferAppender.cs b/src/SeqCli/Forwarder/Storage/BufferAppender.cs index b88576fd..bc2ca6a6 100644 --- a/src/SeqCli/Forwarder/Storage/BufferAppender.cs +++ b/src/SeqCli/Forwarder/Storage/BufferAppender.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ namespace SeqCli.Forwarder.Storage; /// /// The write-side of a buffer. /// -public sealed class BufferAppender : IDisposable +sealed class BufferAppender : IDisposable { readonly StoreDirectory _storeDirectory; BufferAppenderChunk? _currentChunk; @@ -69,19 +69,30 @@ public bool TryAppend(Span batch, long targetChunkLength, int? maxChunks = { if (batch.Length == 0) return true; - if (batch[^1] != (byte)'\n') throw new Exception("Batches must end with a newline character (\\n)"); + if (batch[^1] != (byte)'\n') throw new Exception("Batches must end with a newline character (\\n)."); if (_currentChunk != null) + { // Only use the existing chunk if it's writable and shouldn't be rolled over if (_currentChunk.WriteHead > targetChunkLength) { // Run a sync before moving to a new file, just to make sure any // buffered data makes its way to disk - _currentChunk.Appender.Sync(); - - _currentChunk.Dispose(); - _currentChunk = null; + try + { + _currentChunk.Appender.Sync(); + } + catch (IOException) + { + // Ignored + } + finally + { + _currentChunk.Dispose(); + _currentChunk = null; + } } + } // If there's no suitable candidate chunk then create a new one if (_currentChunk == null) diff --git a/src/SeqCli/Forwarder/Storage/BufferAppenderChunk.cs b/src/SeqCli/Forwarder/Storage/BufferAppenderChunk.cs index f68acd67..69156679 100644 --- a/src/SeqCli/Forwarder/Storage/BufferAppenderChunk.cs +++ b/src/SeqCli/Forwarder/Storage/BufferAppenderChunk.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,8 +17,14 @@ namespace SeqCli.Forwarder.Storage; -sealed record BufferAppenderChunk(StoreFileAppender Appender) : IDisposable +class BufferAppenderChunk : IDisposable { + public BufferAppenderChunk(StoreFileAppender appender) + { + Appender = appender; + } + + public StoreFileAppender Appender { get; } public long WriteHead { get; set; } public void Dispose() diff --git a/src/SeqCli/Forwarder/Storage/BufferReader.cs b/src/SeqCli/Forwarder/Storage/BufferReader.cs index f2746b8c..6fba78e9 100644 --- a/src/SeqCli/Forwarder/Storage/BufferReader.cs +++ b/src/SeqCli/Forwarder/Storage/BufferReader.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace SeqCli.Forwarder.Storage; /// /// The read-side of a buffer. /// -public sealed class BufferReader +sealed class BufferReader { readonly StoreDirectory _storeDirectory; BufferReaderHead? _discardingHead; diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs b/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs index 069c7535..697b98e4 100644 --- a/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs +++ b/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ namespace SeqCli.Forwarder.Storage; /// /// A contiguous batch of records pulled from a reader. /// -public readonly record struct BufferReaderBatch +readonly record struct BufferReaderBatch { readonly int _length; diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderChunk.cs b/src/SeqCli/Forwarder/Storage/BufferReaderChunk.cs index cf7f9b0c..e957f3fc 100644 --- a/src/SeqCli/Forwarder/Storage/BufferReaderChunk.cs +++ b/src/SeqCli/Forwarder/Storage/BufferReaderChunk.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,8 +20,17 @@ namespace SeqCli.Forwarder.Storage; /// /// An active chunk in a . /// -record BufferReaderChunk(ChunkName Name, StoreFile Chunk) : IDisposable +class BufferReaderChunk : IDisposable { + public BufferReaderChunk(ChunkName name, StoreFile chunk) + { + Name = name; + Chunk = chunk; + } + + public ChunkName Name { get; } + public StoreFile Chunk { get; } + (long, StoreFileReader)? _reader; public void Dispose() diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderChunkHead.cs b/src/SeqCli/Forwarder/Storage/BufferReaderChunkHead.cs index a1778342..7969d254 100644 --- a/src/SeqCli/Forwarder/Storage/BufferReaderChunkHead.cs +++ b/src/SeqCli/Forwarder/Storage/BufferReaderChunkHead.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ namespace SeqCli.Forwarder.Storage; /// /// The current position in a . /// -public readonly record struct BufferReaderChunkHead(long CommitHead, long WriteHead) +readonly record struct BufferReaderChunkHead(long CommitHead, long WriteHead) { public long Unadvanced => WriteHead - CommitHead; } diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderHead.cs b/src/SeqCli/Forwarder/Storage/BufferReaderHead.cs index 0240fa51..f1f34217 100644 --- a/src/SeqCli/Forwarder/Storage/BufferReaderHead.cs +++ b/src/SeqCli/Forwarder/Storage/BufferReaderHead.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,4 +17,4 @@ namespace SeqCli.Forwarder.Storage; /// /// A position in a . /// -public readonly record struct BufferReaderHead(ulong Chunk, long CommitHead); +readonly record struct BufferReaderHead(ulong Chunk, long CommitHead); diff --git a/src/SeqCli/Forwarder/Storage/ChunkName.cs b/src/SeqCli/Forwarder/Storage/ChunkName.cs index fc301cf4..dcc85a78 100644 --- a/src/SeqCli/Forwarder/Storage/ChunkName.cs +++ b/src/SeqCli/Forwarder/Storage/ChunkName.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ namespace SeqCli.Forwarder.Storage; /// /// A chunk file name with its incrementing identifier. /// -public readonly record struct ChunkName +readonly record struct ChunkName { readonly string _name; diff --git a/src/SeqCli/Forwarder/Storage/Identifier.cs b/src/SeqCli/Forwarder/Storage/Identifier.cs index 000e0f07..0f0ab980 100644 --- a/src/SeqCli/Forwarder/Storage/Identifier.cs +++ b/src/SeqCli/Forwarder/Storage/Identifier.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ namespace SeqCli.Forwarder.Storage; /// /// Utilities for parsing and formatting file names with sortable identifiers. /// -public static class Identifier +static class Identifier { /// /// Try parse the identifier from the given name with the given extension. diff --git a/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs index 822ecb23..60b418f6 100644 --- a/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index e98c3828..78194d2e 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs index 2cbb3f8f..b9fe5686 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Forwarder/Web/Host/ServerService.cs b/src/SeqCli/Forwarder/Web/Host/ServerService.cs index 6832e953..a8fb7e75 100644 --- a/src/SeqCli/Forwarder/Web/Host/ServerService.cs +++ b/src/SeqCli/Forwarder/Web/Host/ServerService.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Sample/Loader/Simulation.cs b/src/SeqCli/Sample/Loader/Simulation.cs index fba0a939..196ea265 100644 --- a/src/SeqCli/Sample/Loader/Simulation.cs +++ b/src/SeqCli/Sample/Loader/Simulation.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index c15f93fb..97479768 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -30,6 +30,9 @@ LINUX + + UNIX + diff --git a/src/SeqCli/Templates/Ast/JsonTemplate.cs b/src/SeqCli/Templates/Ast/JsonTemplate.cs index 4105f2de..06d74822 100644 --- a/src/SeqCli/Templates/Ast/JsonTemplate.cs +++ b/src/SeqCli/Templates/Ast/JsonTemplate.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Ast/JsonTemplateArray.cs b/src/SeqCli/Templates/Ast/JsonTemplateArray.cs index d7224419..cfd63080 100644 --- a/src/SeqCli/Templates/Ast/JsonTemplateArray.cs +++ b/src/SeqCli/Templates/Ast/JsonTemplateArray.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Ast/JsonTemplateBoolean.cs b/src/SeqCli/Templates/Ast/JsonTemplateBoolean.cs index a6599a64..1fc1bf2e 100644 --- a/src/SeqCli/Templates/Ast/JsonTemplateBoolean.cs +++ b/src/SeqCli/Templates/Ast/JsonTemplateBoolean.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Ast/JsonTemplateCall.cs b/src/SeqCli/Templates/Ast/JsonTemplateCall.cs index ddcb93bc..1f1cd516 100644 --- a/src/SeqCli/Templates/Ast/JsonTemplateCall.cs +++ b/src/SeqCli/Templates/Ast/JsonTemplateCall.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Ast/JsonTemplateNull.cs b/src/SeqCli/Templates/Ast/JsonTemplateNull.cs index dd9cda6c..01a44a6e 100644 --- a/src/SeqCli/Templates/Ast/JsonTemplateNull.cs +++ b/src/SeqCli/Templates/Ast/JsonTemplateNull.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Ast/JsonTemplateNumber.cs b/src/SeqCli/Templates/Ast/JsonTemplateNumber.cs index bb149866..32887fc7 100644 --- a/src/SeqCli/Templates/Ast/JsonTemplateNumber.cs +++ b/src/SeqCli/Templates/Ast/JsonTemplateNumber.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Ast/JsonTemplateObject.cs b/src/SeqCli/Templates/Ast/JsonTemplateObject.cs index 0c5be41b..fe11aac5 100644 --- a/src/SeqCli/Templates/Ast/JsonTemplateObject.cs +++ b/src/SeqCli/Templates/Ast/JsonTemplateObject.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Ast/JsonTemplateString.cs b/src/SeqCli/Templates/Ast/JsonTemplateString.cs index 96d2e8d6..d7896871 100644 --- a/src/SeqCli/Templates/Ast/JsonTemplateString.cs +++ b/src/SeqCli/Templates/Ast/JsonTemplateString.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Evaluator/JsonTemplateEvaluator.cs b/src/SeqCli/Templates/Evaluator/JsonTemplateEvaluator.cs index 4bb07eba..977b476e 100644 --- a/src/SeqCli/Templates/Evaluator/JsonTemplateEvaluator.cs +++ b/src/SeqCli/Templates/Evaluator/JsonTemplateEvaluator.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Evaluator/JsonTemplateFunction.cs b/src/SeqCli/Templates/Evaluator/JsonTemplateFunction.cs index 173bacd6..9d06abb9 100644 --- a/src/SeqCli/Templates/Evaluator/JsonTemplateFunction.cs +++ b/src/SeqCli/Templates/Evaluator/JsonTemplateFunction.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Import/EntityTemplate.cs b/src/SeqCli/Templates/Import/EntityTemplate.cs index eb99d3e9..fa08bde5 100644 --- a/src/SeqCli/Templates/Import/EntityTemplate.cs +++ b/src/SeqCli/Templates/Import/EntityTemplate.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Import/EntityTemplateLoader.cs b/src/SeqCli/Templates/Import/EntityTemplateLoader.cs index 39a0f857..92975cdf 100644 --- a/src/SeqCli/Templates/Import/EntityTemplateLoader.cs +++ b/src/SeqCli/Templates/Import/EntityTemplateLoader.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Import/GenericEntity.cs b/src/SeqCli/Templates/Import/GenericEntity.cs index 48838362..b9f93650 100644 --- a/src/SeqCli/Templates/Import/GenericEntity.cs +++ b/src/SeqCli/Templates/Import/GenericEntity.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Import/TemplateSetImporter.cs b/src/SeqCli/Templates/Import/TemplateSetImporter.cs index 399e4bcf..9f14ec5c 100644 --- a/src/SeqCli/Templates/Import/TemplateSetImporter.cs +++ b/src/SeqCli/Templates/Import/TemplateSetImporter.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/ObjectGraphs/JsonTemplateObjectGraphConverter.cs b/src/SeqCli/Templates/ObjectGraphs/JsonTemplateObjectGraphConverter.cs index 9236e260..c23c8d2c 100644 --- a/src/SeqCli/Templates/ObjectGraphs/JsonTemplateObjectGraphConverter.cs +++ b/src/SeqCli/Templates/ObjectGraphs/JsonTemplateObjectGraphConverter.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Parser/JsonTemplateParser.cs b/src/SeqCli/Templates/Parser/JsonTemplateParser.cs index 5a24778f..a0cdd15b 100644 --- a/src/SeqCli/Templates/Parser/JsonTemplateParser.cs +++ b/src/SeqCli/Templates/Parser/JsonTemplateParser.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Parser/JsonTemplateTextParsers.cs b/src/SeqCli/Templates/Parser/JsonTemplateTextParsers.cs index cf28a346..41746e39 100644 --- a/src/SeqCli/Templates/Parser/JsonTemplateTextParsers.cs +++ b/src/SeqCli/Templates/Parser/JsonTemplateTextParsers.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Parser/JsonTemplateToken.cs b/src/SeqCli/Templates/Parser/JsonTemplateToken.cs index 289b48b1..fb375455 100644 --- a/src/SeqCli/Templates/Parser/JsonTemplateToken.cs +++ b/src/SeqCli/Templates/Parser/JsonTemplateToken.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Parser/JsonTemplateTokenizer.cs b/src/SeqCli/Templates/Parser/JsonTemplateTokenizer.cs index ed3c01f1..b2da6ca4 100644 --- a/src/SeqCli/Templates/Parser/JsonTemplateTokenizer.cs +++ b/src/SeqCli/Templates/Parser/JsonTemplateTokenizer.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Util/LogEventPropertyFactory.cs b/src/SeqCli/Util/LogEventPropertyFactory.cs index 6b2a004d..89c23987 100644 --- a/src/SeqCli/Util/LogEventPropertyFactory.cs +++ b/src/SeqCli/Util/LogEventPropertyFactory.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreDirectory.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreDirectory.cs index 7382bc1d..00b1762d 100644 --- a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreDirectory.cs +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreDirectory.cs @@ -5,7 +5,7 @@ namespace SeqCli.Tests.Forwarder.Filesystem; -public class InMemoryStoreDirectory : StoreDirectory +class InMemoryStoreDirectory : StoreDirectory { readonly Dictionary _files = new(); @@ -13,7 +13,7 @@ public class InMemoryStoreDirectory : StoreDirectory public override InMemoryStoreFile Create(string name) { - if (_files.ContainsKey(name)) throw new Exception($"The file {name} already exists"); + if (_files.ContainsKey(name)) throw new Exception($"The file {name} already exists."); _files.Add(name, new InMemoryStoreFile()); diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs index dd2b54e8..26be3beb 100644 --- a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs @@ -6,7 +6,7 @@ namespace SeqCli.Tests.Forwarder.Filesystem; -public class InMemoryStoreFile : StoreFile +class InMemoryStoreFile : StoreFile { public byte[] Contents { get; private set; } = Array.Empty(); diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileAppender.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileAppender.cs index aeaf66e1..9b078eec 100644 --- a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileAppender.cs +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileAppender.cs @@ -4,7 +4,7 @@ namespace SeqCli.Tests.Forwarder.Filesystem; -public class InMemoryStoreFileAppender : StoreFileAppender +class InMemoryStoreFileAppender : StoreFileAppender { readonly List _incoming; diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileReader.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileReader.cs index 56bd0838..d8507e6c 100644 --- a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileReader.cs +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileReader.cs @@ -3,7 +3,7 @@ namespace SeqCli.Tests.Forwarder.Filesystem; -public class InMemoryStoreFileReader : StoreFileReader +class InMemoryStoreFileReader : StoreFileReader { readonly int _length; From ad3a7c95d8c458ff6911195fa904b76e28b294a1 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 17 Jul 2025 20:39:15 +1000 Subject: [PATCH 38/51] Simple hook-up of API to buffer writer --- seqcli.sln.DotSettings | 1 + .../Cli/Commands/Forwarder/RunCommand.cs | 2 + .../Forwarder/ForwarderDiagnosticConfig.cs | 1 + src/SeqCli/Forwarder/Channel/LogChannel.cs | 6 +-- src/SeqCli/Forwarder/Channel/LogChannelMap.cs | 25 +++++++-- .../Forwarder/Filesystem/StoreDirectory.cs | 2 +- .../Filesystem/System/SystemStoreDirectory.cs | 2 +- src/SeqCli/Forwarder/ForwarderModule.cs | 8 ++- src/SeqCli/Forwarder/Storage/Bookmark.cs | 4 +- .../Forwarder/Web/Api/IngestionEndpoints.cs | 52 ++++++++----------- .../Web/Api/IngestionLogEndpoints.cs | 5 +- src/SeqCli/Properties/launchSettings.json | 2 +- 12 files changed, 62 insertions(+), 48 deletions(-) diff --git a/seqcli.sln.DotSettings b/seqcli.sln.DotSettings index 6c218398..95d327ba 100644 --- a/seqcli.sln.DotSettings +++ b/seqcli.sln.DotSettings @@ -11,6 +11,7 @@ True True True + True True True True diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index dc2d6a54..72928376 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -89,6 +89,8 @@ protected override async Task Run(string[] unrecognized) logger.Fatal(ex, "Failed to load configuration from {ConfigFilePath}", _storagePath.ConfigFilePath); return 1; } + + Log.Information("Loaded configuration from {ConfigFilePath}", _storagePath.ConfigFilePath); Log.Logger = CreateLogger( config.Forwarder.Diagnostics.InternalLoggingLevel, diff --git a/src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs index 3a63d685..6a9fab48 100644 --- a/src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs +++ b/src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs @@ -12,6 +12,7 @@ public class ForwarderDiagnosticConfig public LogEventLevel InternalLoggingLevel { get; set; } = LogEventLevel.Information; public string? InternalLogServerUri { get; set; } public string? InternalLogServerApiKey { get; set; } + public bool ExposeIngestionLog { get; set; } public bool IngestionLogShowDetail { get; set; } public static string GetDefaultInternalLogPath() diff --git a/src/SeqCli/Forwarder/Channel/LogChannel.cs b/src/SeqCli/Forwarder/Channel/LogChannel.cs index 4a02585b..c1567813 100644 --- a/src/SeqCli/Forwarder/Channel/LogChannel.cs +++ b/src/SeqCli/Forwarder/Channel/LogChannel.cs @@ -7,7 +7,7 @@ namespace SeqCli.Forwarder.Channel; class LogChannel { - public LogChannel(Func write, CancellationToken cancellationToken) + public LogChannel(Func, CancellationToken, Task> write) { var channel = System.Threading.Channels.Channel.CreateBounded(new BoundedChannelOptions(5) { @@ -16,7 +16,7 @@ public LogChannel(Func write, CancellationToken cancell FullMode = BoundedChannelFullMode.Wait, }); - _shutdownTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + _shutdownTokenSource = new CancellationTokenSource(); _writer = channel.Writer; _worker = Task.Run(async () => { @@ -24,7 +24,7 @@ public LogChannel(Func write, CancellationToken cancell { try { - await write(_shutdownTokenSource.Token); + await write(entry.Storage[entry.Range], _shutdownTokenSource.Token); entry.Completion.SetResult(); } catch (Exception e) diff --git a/src/SeqCli/Forwarder/Channel/LogChannelMap.cs b/src/SeqCli/Forwarder/Channel/LogChannelMap.cs index 3bb24e90..df7c3dd9 100644 --- a/src/SeqCli/Forwarder/Channel/LogChannelMap.cs +++ b/src/SeqCli/Forwarder/Channel/LogChannelMap.cs @@ -1,24 +1,41 @@ using System; +using System.IO; using System.Threading.Tasks; +using SeqCli.Forwarder.Filesystem.System; +using SeqCli.Forwarder.Storage; using Serilog; namespace SeqCli.Forwarder.Channel; class LogChannelMap { - public LogChannelMap() + readonly BufferAppender _defaultAppender; + readonly LogChannel _defaultChannel; + + public LogChannelMap(string bufferPath) { + var defaultStore = new SystemStoreDirectory(bufferPath); + Log.Information("Opening local buffer in {BufferPath}", bufferPath); + _defaultAppender = BufferAppender.Open(defaultStore); + _defaultChannel = new LogChannel((chunk, _) => + { + // TODO: chunk sizes, max chunks, ingestion log + _defaultAppender.TryAppend(chunk.AsSpan(), 100_000_000); + return Task.CompletedTask; + }); } public LogChannel Get(string? apiKey) { - return new LogChannel(async (c) => await Task.Delay(TimeSpan.FromSeconds(1), c), default); + // apiKey is ignored. + return _defaultChannel; } - public Task StopAsync() + public async Task StopAsync() { Log.Information("Flushing log buffers"); - return Task.CompletedTask; + await _defaultChannel.StopAsync(); + _defaultAppender.Dispose(); } } diff --git a/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs b/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs index 39008865..f6f34d36 100644 --- a/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs +++ b/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs @@ -29,7 +29,7 @@ abstract class StoreDirectory /// public abstract StoreFile Create(string name); - public virtual (string, StoreFile) CreateTemporary() + protected virtual (string, StoreFile) CreateTemporary() { var tmpName = $"rc{Guid.NewGuid():N}.tmp"; return (tmpName, Create(tmpName)); diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs index e436185f..c1998e73 100644 --- a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs @@ -44,7 +44,7 @@ public override SystemStoreFile Create(string name) return new SystemStoreFile(filePath); } - public override (string, StoreFile) CreateTemporary() + protected override (string, StoreFile) CreateTemporary() { // Temporary files are still created in the same directory // This is necessary for renames to be atomic on some filesystems diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 0cc9c954..72f76e64 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -38,10 +38,16 @@ public ForwarderModule(string bufferPath, SeqCliConfig config) protected override void Load(ContainerBuilder builder) { builder.RegisterType().SingleInstance(); - builder.RegisterType().SingleInstance(); + builder.Register(_ => new LogChannelMap(_bufferPath)).SingleInstance(); builder.RegisterType().As(); builder.RegisterType().As(); + + if (_config.Forwarder.Diagnostics.ExposeIngestionLog) + { + builder.RegisterType().As(); + } + builder.RegisterInstance(new MessageTemplateTextFormatter( "[{Timestamp:o} {Level:u3}] {Message}{NewLine}" + (_config.Forwarder.Diagnostics.IngestionLogShowDetail ? "" diff --git a/src/SeqCli/Forwarder/Storage/Bookmark.cs b/src/SeqCli/Forwarder/Storage/Bookmark.cs index 3f75fb8b..44d0834d 100644 --- a/src/SeqCli/Forwarder/Storage/Bookmark.cs +++ b/src/SeqCli/Forwarder/Storage/Bookmark.cs @@ -17,6 +17,7 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; +using System.Threading; using SeqCli.Forwarder.Filesystem; using Path = System.IO.Path; @@ -29,9 +30,8 @@ sealed class Bookmark { readonly StoreDirectory _storeDirectory; - readonly object _sync = new(); + readonly Lock _sync = new(); BookmarkName _name; - BookmarkValue? _value; Bookmark(StoreDirectory storeDirectory, BookmarkName name, BookmarkValue? value) diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index dd4e597a..e75655a7 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -14,9 +14,7 @@ using System; using System.Buffers; -using System.Collections.Generic; using System.Linq; -using System.Net; using System.Text; using System.Text.Json; using System.Threading; @@ -25,55 +23,47 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.Net.Http.Headers; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using SeqCli.Config; using SeqCli.Forwarder.Channel; using SeqCli.Forwarder.Diagnostics; using JsonException = System.Text.Json.JsonException; -using JsonSerializer = Newtonsoft.Json.JsonSerializer; namespace SeqCli.Forwarder.Web.Api; +// ReSharper disable UnusedMethodReturnValue.Local + class IngestionEndpoints : IMapEndpoints { static readonly Encoding Utf8 = new UTF8Encoding(false); - readonly SeqCliConnectionConfig _seqCliConnectionConfig; readonly LogChannelMap _logChannels; - readonly JsonSerializer _rawSerializer = JsonSerializer.Create( - new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); - - public IngestionEndpoints( - SeqCliConfig config, - LogChannelMap logChannels) + public IngestionEndpoints(LogChannelMap logChannels) { - _seqCliConnectionConfig = config.Connection; _logChannels = logChannels; } public void MapEndpoints(WebApplication app) { - app.MapPost("api/events/raw", new Func>(async (context) => - { - var clef = DefaultedBoolQuery(context.Request, "clef"); + app.MapPost("ingest/clef", async context => await IngestCompactFormatAsync(context)); + app.MapPost("api/events/raw", async context => await IngestAsync(context)); + } - if (clef) - return await IngestCompactFormat(context); + async Task IngestAsync(HttpContext context) + { + var clef = DefaultedBoolQuery(context.Request, "clef"); + + if (clef) return await IngestCompactFormatAsync(context); - var contentType = (string?) context.Request.Headers[HeaderNames.ContentType]; - const string clefMediaType = "application/vnd.serilog.clef"; + var contentType = (string?)context.Request.Headers[HeaderNames.ContentType]; + const string clefMediaType = "application/vnd.serilog.clef"; - if (contentType != null && contentType.StartsWith(clefMediaType)) - return await IngestCompactFormat(context); + if (contentType != null && contentType.StartsWith(clefMediaType)) return await IngestCompactFormatAsync(context); - IngestionLog.ForClient(context.Connection.RemoteIpAddress) - .Error("Client supplied a legacy raw-format (non-CLEF) payload"); - return Results.BadRequest("Only newline-delimited JSON (CLEF) payloads are supported."); - })); + IngestionLog.ForClient(context.Connection.RemoteIpAddress) + .Error("Client supplied a legacy raw-format (non-CLEF) payload"); + return Results.BadRequest("Only newline-delimited JSON (CLEF) payloads are supported."); } - + static bool DefaultedBoolQuery(HttpRequest request, string queryParameterName) { var parameter = request.Query[queryParameterName]; @@ -92,7 +82,7 @@ static bool DefaultedBoolQuery(HttpRequest request, string queryParameterName) return "true".Equals(value, StringComparison.OrdinalIgnoreCase) || value == "" || value == queryParameterName; } - static string? ApiKey(HttpRequest request) + static string? GetApiKey(HttpRequest request) { var apiKeyHeader = request.Headers["X-SeqApiKey"]; @@ -100,12 +90,12 @@ static bool DefaultedBoolQuery(HttpRequest request, string queryParameterName) return request.Query.TryGetValue("apiKey", out var apiKey) ? apiKey.Last() : null; } - async Task IngestCompactFormat(HttpContext context) + async Task IngestCompactFormatAsync(HttpContext context) { var cts = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted); cts.CancelAfter(TimeSpan.FromSeconds(5)); - var log = _logChannels.Get(ApiKey(context.Request)); + var log = _logChannels.Get(GetApiKey(context.Request)); var payload = ArrayPool.Shared.Rent(1024 * 1024 * 10); var writeHead = 0; diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs index b9fe5686..60a69038 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs @@ -33,10 +33,7 @@ public IngestionLogEndpoints(MessageTemplateTextFormatter formatter) public void MapEndpoints(WebApplication app) { - // ISSUE: this route should probably only be mapped when some kind of --unsafe-debug flag - // is set. - - app.MapGet("/", () => + app.MapGet("api/diagnostics/ingestion", () => { var events = IngestionLog.Read(); using var log = new StringWriter(); diff --git a/src/SeqCli/Properties/launchSettings.json b/src/SeqCli/Properties/launchSettings.json index fbea9f6a..f9a7cad5 100644 --- a/src/SeqCli/Properties/launchSettings.json +++ b/src/SeqCli/Properties/launchSettings.json @@ -3,7 +3,7 @@ "profiles": { "SeqCli": { "commandName": "Project", - "commandLineArgs": "config -k connection.apiKey -v test" + "commandLineArgs": "forwarder run --pre" } } } From f79d4358abf2675682ce074acc06691f56fa08ce Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 18 Jul 2025 12:39:16 +1000 Subject: [PATCH 39/51] Reorganize a little, get basic forwarding working --- src/SeqCli/Cli/CommandLineHost.cs | 13 ++- src/SeqCli/Cli/Commands/ConfigCommand.cs | 16 ++-- .../Cli/Commands/Forwarder/InstallCommand.cs | 4 +- .../Cli/Commands/Forwarder/RunCommand.cs | 24 +++-- .../Cli/Commands/Forwarder/TruncateCommand.cs | 4 +- src/SeqCli/Cli/Commands/IngestCommand.cs | 8 +- .../Cli/Commands/Profile/CreateCommand.cs | 10 +- .../Cli/Commands/Profile/ListCommand.cs | 10 +- .../Cli/Commands/Profile/RemoveCommand.cs | 10 +- .../Features/SendFailureHandlingFeature.cs | 2 +- src/SeqCli/Cli/Features/StoragePathFeature.cs | 18 +--- .../Config/RuntimeConfigurationLoader.cs | 10 +- .../Forwarder/Channel/ForwardingChannel.cs | 96 +++++++++++++++++++ .../Channel/ForwardingChannelEntry.cs | 6 ++ .../Forwarder/Channel/ForwardingChannelMap.cs | 90 +++++++++++++++++ src/SeqCli/Forwarder/Channel/LogChannel.cs | 57 ----------- .../Forwarder/Channel/LogChannelEntry.cs | 6 -- src/SeqCli/Forwarder/Channel/LogChannelMap.cs | 41 -------- src/SeqCli/Forwarder/ForwarderModule.cs | 9 +- .../Forwarder/Storage/BufferReaderBatch.cs | 5 + .../Forwarder/Web/Api/IngestionEndpoints.cs | 15 +-- .../Forwarder/Web/Host/ServerService.cs | 8 +- src/SeqCli/Ingestion/LogShipper.cs | 56 +++++++++++ src/SeqCli/Program.cs | 1 + src/SeqCli/SeqCliModule.cs | 4 +- test/SeqCli.Tests/Cli/CommandLineHostTests.cs | 9 +- 26 files changed, 358 insertions(+), 174 deletions(-) create mode 100644 src/SeqCli/Forwarder/Channel/ForwardingChannel.cs create mode 100644 src/SeqCli/Forwarder/Channel/ForwardingChannelEntry.cs create mode 100644 src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs delete mode 100644 src/SeqCli/Forwarder/Channel/LogChannel.cs delete mode 100644 src/SeqCli/Forwarder/Channel/LogChannelEntry.cs delete mode 100644 src/SeqCli/Forwarder/Channel/LogChannelMap.cs diff --git a/src/SeqCli/Cli/CommandLineHost.cs b/src/SeqCli/Cli/CommandLineHost.cs index 407282ef..0c156456 100644 --- a/src/SeqCli/Cli/CommandLineHost.cs +++ b/src/SeqCli/Cli/CommandLineHost.cs @@ -18,6 +18,7 @@ using System.Reflection; using System.Threading.Tasks; using Autofac.Features.Metadata; +using SeqCli.Cli.Features; using Serilog.Core; using Serilog.Events; @@ -25,10 +26,12 @@ namespace SeqCli.Cli; class CommandLineHost { + readonly StoragePathFeature _storagePathFeature; readonly List, CommandMetadata>> _availableCommands; - public CommandLineHost(IEnumerable, CommandMetadata>> availableCommands) + public CommandLineHost(IEnumerable, CommandMetadata>> availableCommands, StoragePathFeature storagePathFeature) { + _storagePathFeature = storagePathFeature; _availableCommands = availableCommands.ToList(); } @@ -63,7 +66,13 @@ public async Task Run(string[] args, LoggingLevelSwitch levelSwitch) commandSpecificArgs = commandSpecificArgs.Where(arg => arg != verboseArg).ToArray(); } - return await cmd.Value.Value.Invoke(commandSpecificArgs); + var impl = cmd.Value.Value; + + // This one is global, because implicitly-created components in the container rely on the specified + // storage path. It's a convoluted data flow that we should rip out when possible. + _storagePathFeature.Enable(impl.Options); + + return await impl.Invoke(commandSpecificArgs); } } diff --git a/src/SeqCli/Cli/Commands/ConfigCommand.cs b/src/SeqCli/Cli/Commands/ConfigCommand.cs index c4b1142b..2e71f07e 100644 --- a/src/SeqCli/Cli/Commands/ConfigCommand.cs +++ b/src/SeqCli/Cli/Commands/ConfigCommand.cs @@ -13,12 +13,8 @@ // limitations under the License. using System; -using System.Collections; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Reflection; using System.Threading.Tasks; +using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Util; using Serilog; @@ -30,12 +26,14 @@ class ConfigCommand : Command { string? _key, _value; bool _clear; + readonly StoragePathFeature _storagePath; - public ConfigCommand() + public ConfigCommand(StoragePathFeature storagePath) { Options.Add("k|key=", "The field, for example `connection.serverUrl`", k => _key = k); Options.Add("v|value=", "The field value; if not specified, the command will print the current value", v => _value = v); Options.Add("c|clear", "Clear the field", _ => _clear = true); + _storagePath = storagePath; } protected override Task Run() @@ -44,7 +42,7 @@ protected override Task Run() try { - var config = SeqCliConfig.ReadFromFile(RuntimeConfigurationLoader.DefaultConfigFilename); + var config = SeqCliConfig.ReadFromFile(_storagePath.ConfigFilePath); if (_key != null) { @@ -52,13 +50,13 @@ protected override Task Run() { verb = "clear"; KeyValueSettings.Clear(config, _key); - SeqCliConfig.WriteToFile(config, RuntimeConfigurationLoader.DefaultConfigFilename); + SeqCliConfig.WriteToFile(config, _storagePath.ConfigFilePath); } else if (_value != null) { verb = "update"; KeyValueSettings.Set(config, _key, _value); - SeqCliConfig.WriteToFile(config, RuntimeConfigurationLoader.DefaultConfigFilename); + SeqCliConfig.WriteToFile(config, _storagePath.ConfigFilePath); } else { diff --git a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs index 4bb3a21f..63c8a2ee 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs @@ -41,9 +41,9 @@ class InstallCommand : Command bool _setup; - public InstallCommand() + public InstallCommand(StoragePathFeature storagePath) { - _storagePath = Enable(); + _storagePath = storagePath; _listenUri = Enable(); _serviceCredentials = Enable(); diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index 72928376..4b45e667 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -26,6 +26,7 @@ using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Config.Forwarder; +using SeqCli.Connection; using SeqCli.Forwarder; using SeqCli.Forwarder.Util; using SeqCli.Forwarder.Web.Api; @@ -47,16 +48,20 @@ namespace SeqCli.Cli.Commands.Forwarder; [Command("forwarder", "run", "Listen on an HTTP endpoint and forward ingested logs to Seq", IsPreview = true)] class RunCommand : Command { + readonly SeqConnectionFactory _connectionFactory; readonly StoragePathFeature _storagePath; readonly ListenUriFeature _listenUri; + readonly ConnectionFeature _connection; bool _noLogo; - public RunCommand() + public RunCommand(SeqConnectionFactory connectionFactory, StoragePathFeature storagePath) { + _connectionFactory = connectionFactory; Options.Add("nologo", _ => _noLogo = true); - _storagePath = Enable(); _listenUri = Enable(); + _connection = Enable(); + _storagePath = storagePath; } protected override async Task Run(string[] unrecognized) @@ -74,7 +79,6 @@ protected override async Task Run(string[] unrecognized) } SeqCliConfig config; - try { // ISSUE: we can't really rely on the default `SeqCliConfig` path being readable when running as a service. @@ -90,14 +94,22 @@ protected override async Task Run(string[] unrecognized) return 1; } - Log.Information("Loaded configuration from {ConfigFilePath}", _storagePath.ConfigFilePath); - + var connection = _connectionFactory.Connect(_connection); + + // The API key is passed through separately because `SeqConnection` doesn't expose a batched ingestion + // mechanism and so we manually construct `HttpRequestMessage`s deeper in the stack. Nice feature gap to + // close at some point! + var (serverUrl, apiKey) = _connectionFactory.GetConnectionDetails(_connection); + Log.Logger = CreateLogger( config.Forwarder.Diagnostics.InternalLoggingLevel, config.Forwarder.Diagnostics.InternalLogPath, config.Forwarder.Diagnostics.InternalLogServerUri, config.Forwarder.Diagnostics.InternalLogServerApiKey); + Log.Information("Loaded configuration from {ConfigFilePath}", _storagePath.ConfigFilePath); + Log.Information("Forwarding to {ServerUrl}", serverUrl); + var listenUri = _listenUri.ListenUri ?? config.Forwarder.Api.ListenUri; try @@ -145,7 +157,7 @@ protected override async Task Run(string[] unrecognized) .ConfigureContainer(containerBuilder => { containerBuilder.RegisterBuildCallback(ls => container = ls); - containerBuilder.RegisterModule(new ForwarderModule(_storagePath.BufferPath, config)); + containerBuilder.RegisterModule(new ForwarderModule(_storagePath.BufferPath, config, connection, apiKey)); }); await using var app = builder.Build(); diff --git a/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs index bf50a2ae..08a8a678 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs @@ -25,9 +25,9 @@ class TruncateCommand : Command readonly StoragePathFeature _storagePath; readonly ConfirmFeature _confirm; - public TruncateCommand() + public TruncateCommand(StoragePathFeature storagePath) { - _storagePath = Enable(); + _storagePath = storagePath; _confirm = Enable(); } diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index ad52e0ba..22f4a181 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -98,6 +98,10 @@ protected override async Task Run() } var connection = _connectionFactory.Connect(_connection); + + // The API key is passed through separately because `SeqConnection` doesn't expose a batched ingestion + // mechanism and so we manually construct `HttpRequestMessage`s deeper in the stack. Nice feature gap to + // close at some point! var (_, apiKey) = _connectionFactory.GetConnectionDetails(_connection); var batchSize = _batchSize.Value; @@ -105,8 +109,8 @@ protected override async Task Run() { using (input) { - var reader = _json - ? (ILogEventReader) new JsonLogEventReader(input) + ILogEventReader reader = _json + ? new JsonLogEventReader(input) : new PlainTextLogEventReader(input, _pattern); reader = new EnrichingReader(reader, enrichers); diff --git a/src/SeqCli/Cli/Commands/Profile/CreateCommand.cs b/src/SeqCli/Cli/Commands/Profile/CreateCommand.cs index 0a6f7140..e5be2268 100644 --- a/src/SeqCli/Cli/Commands/Profile/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Profile/CreateCommand.cs @@ -1,5 +1,6 @@ using System; using System.Threading.Tasks; +using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Util; using Serilog; @@ -11,8 +12,9 @@ namespace SeqCli.Cli.Commands.Profile; class CreateCommand : Command { string? _url, _apiKey, _name; + readonly StoragePathFeature _storagePath; - public CreateCommand() + public CreateCommand(StoragePathFeature storagePath) { Options.Add("n=|name=", "The name of the connection profile", @@ -25,6 +27,8 @@ public CreateCommand() Options.Add("a=|apikey=", "The API key to use when connecting to the server, if required", v => _apiKey = ArgumentString.Normalize(v)); + + _storagePath = storagePath; } protected override Task Run() @@ -48,11 +52,11 @@ int RunSync() try { - var config = SeqCliConfig.ReadFromFile(RuntimeConfigurationLoader.DefaultConfigFilename); + var config = SeqCliConfig.ReadFromFile(_storagePath.ConfigFilePath); var connectionConfig = new SeqCliConnectionConfig { ServerUrl = _url }; connectionConfig.EncodeApiKey(_apiKey, config.Encryption.DataProtector()); config.Profiles[_name] = connectionConfig; - SeqCliConfig.WriteToFile(config, RuntimeConfigurationLoader.DefaultConfigFilename); + SeqCliConfig.WriteToFile(config, _storagePath.ConfigFilePath); return 0; } catch (Exception ex) diff --git a/src/SeqCli/Cli/Commands/Profile/ListCommand.cs b/src/SeqCli/Cli/Commands/Profile/ListCommand.cs index 8d3b8048..fcdff3df 100644 --- a/src/SeqCli/Cli/Commands/Profile/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Profile/ListCommand.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Threading.Tasks; +using SeqCli.Cli.Features; using SeqCli.Config; namespace SeqCli.Cli.Commands.Profile; @@ -9,9 +10,16 @@ namespace SeqCli.Cli.Commands.Profile; Example = "seqcli profile list")] class ListCommand : Command { + readonly StoragePathFeature _storagePath; + + public ListCommand(StoragePathFeature storagePath) + { + _storagePath = storagePath; + } + protected override Task Run() { - var config = RuntimeConfigurationLoader.Load(); + var config = RuntimeConfigurationLoader.Load(_storagePath); foreach (var profile in config.Profiles.OrderBy(p => p.Key)) { diff --git a/src/SeqCli/Cli/Commands/Profile/RemoveCommand.cs b/src/SeqCli/Cli/Commands/Profile/RemoveCommand.cs index a112bdb1..bcd69ea4 100644 --- a/src/SeqCli/Cli/Commands/Profile/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/Profile/RemoveCommand.cs @@ -1,5 +1,6 @@ using System; using System.Threading.Tasks; +using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Util; using Serilog; @@ -11,12 +12,15 @@ namespace SeqCli.Cli.Commands.Profile; class RemoveCommand : Command { string? _name; + readonly StoragePathFeature _storagePath; - public RemoveCommand() + public RemoveCommand(StoragePathFeature storagePathPath) { Options.Add("n=|name=", "The name of the connection profile to remove", v => _name = ArgumentString.Normalize(v)); + + _storagePath = storagePathPath; } protected override Task Run() @@ -34,14 +38,14 @@ int RunSync() try { - var config = SeqCliConfig.ReadFromFile(RuntimeConfigurationLoader.DefaultConfigFilename); + var config = SeqCliConfig.ReadFromFile(_storagePath.ConfigFilePath); if (!config.Profiles.Remove(_name)) { Log.Error("No profile with name {ProfileName} was found", _name); return 1; } - SeqCliConfig.WriteToFile(config, RuntimeConfigurationLoader.DefaultConfigFilename); + SeqCliConfig.WriteToFile(config, _storagePath.ConfigFilePath); return 0; } diff --git a/src/SeqCli/Cli/Features/SendFailureHandlingFeature.cs b/src/SeqCli/Cli/Features/SendFailureHandlingFeature.cs index 624623ef..97892e73 100644 --- a/src/SeqCli/Cli/Features/SendFailureHandlingFeature.cs +++ b/src/SeqCli/Cli/Features/SendFailureHandlingFeature.cs @@ -25,6 +25,6 @@ public override void Enable(OptionSet options) { options.Add("send-failure=", "Specify how connection failures are handled: `fail` (default), `retry`, `continue`, or `ignore`", - v => SendFailureHandling = (SendFailureHandling)Enum.Parse(typeof(SendFailureHandling), v, ignoreCase: true)); + v => SendFailureHandling = Enum.Parse(v, ignoreCase: true)); } } \ No newline at end of file diff --git a/src/SeqCli/Cli/Features/StoragePathFeature.cs b/src/SeqCli/Cli/Features/StoragePathFeature.cs index b129f796..990b09ec 100644 --- a/src/SeqCli/Cli/Features/StoragePathFeature.cs +++ b/src/SeqCli/Cli/Features/StoragePathFeature.cs @@ -22,13 +22,13 @@ public string StorageRootPath } } - public string ConfigFilePath => Path.Combine(StorageRootPath, "SeqForwarder.json"); + public string ConfigFilePath => Path.Combine(StorageRootPath, "SeqCli.json"); - public string BufferPath => Path.Combine(StorageRootPath, "Buffer"); + public string BufferPath => Path.Combine(StorageRootPath, "SeqCli", "Buffer"); public override void Enable(OptionSet options) { - options.Add("s=|storage=", + options.Add("storage=", "Set the folder where data will be stored; " + "" + GetDefaultStorageRoot() + " is used by default.", v => _storageRoot = Path.GetFullPath(v)); @@ -36,17 +36,7 @@ public override void Enable(OptionSet options) static string GetDefaultStorageRoot() { - return Path.GetFullPath(Path.Combine( -#if WINDOWS - // Common, here, because the service may run as Local Service, which has no obvious home - // directory. - Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), -#else - // Specific to and writable by the current user. - Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), -#endif - "SeqCli", - "Forwarder")); + return Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); } static string? TryQueryInstalledStorageRoot() diff --git a/src/SeqCli/Config/RuntimeConfigurationLoader.cs b/src/SeqCli/Config/RuntimeConfigurationLoader.cs index 1550e7aa..9dee4560 100644 --- a/src/SeqCli/Config/RuntimeConfigurationLoader.cs +++ b/src/SeqCli/Config/RuntimeConfigurationLoader.cs @@ -12,25 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; -using System.IO; +using SeqCli.Cli.Features; namespace SeqCli.Config; static class RuntimeConfigurationLoader { - public static readonly string DefaultConfigFilename = - Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "SeqCli.json"); - const string DefaultEnvironmentVariablePrefix = "SEQCLI_"; /// /// This is the method to use when loading configuration for runtime use. It will read the default configuration /// file, if any, and apply overrides from the environment. /// - public static SeqCliConfig Load() + public static SeqCliConfig Load(StoragePathFeature storage) { - var config = SeqCliConfig.ReadFromFile(DefaultConfigFilename); + var config = SeqCliConfig.ReadFromFile(storage.ConfigFilePath); EnvironmentOverrides.Apply(DefaultEnvironmentVariablePrefix, config); diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannel.cs b/src/SeqCli/Forwarder/Channel/ForwardingChannel.cs new file mode 100644 index 00000000..cc3cc480 --- /dev/null +++ b/src/SeqCli/Forwarder/Channel/ForwardingChannel.cs @@ -0,0 +1,96 @@ +using System; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using Seq.Api; +using SeqCli.Forwarder.Storage; +using SeqCli.Ingestion; + +namespace SeqCli.Forwarder.Channel; + +class ForwardingChannel +{ + readonly ChannelWriter _writer; + readonly Task _writeWorker, _readWorker; + readonly CancellationTokenSource _stop; + readonly CancellationToken _hardCancel; + + public ForwardingChannel(BufferAppender appender, BufferReader reader, Bookmark bookmark, SeqConnection connection, string? apiKey, CancellationToken hardCancel) + { + var channel = System.Threading.Channels.Channel.CreateBounded(new BoundedChannelOptions(5) + { + SingleReader = false, + SingleWriter = true, + FullMode = BoundedChannelFullMode.Wait, + }); + + _stop = CancellationTokenSource.CreateLinkedTokenSource(_hardCancel); + _hardCancel = hardCancel; + _writer = channel.Writer; + _writeWorker = Task.Run(async () => + { + await foreach (var entry in channel.Reader.ReadAllAsync(hardCancel)) + { + try + { + // TODO: chunk sizes, max chunks, ingestion log + appender.TryAppend(entry.Data.AsSpan(), 100_000_000); + entry.CompletionSource.SetResult(); + } + catch (Exception e) + { + entry.CompletionSource.TrySetException(e); + } + } + }, cancellationToken: hardCancel); + + _readWorker = Task.Run(async () => + { + if (bookmark.TryGet(out var bookmarkValue)) + { + // TODO: initialize reader + // reader.AdvanceTo(bookmarkValue.Value); + } + + while (true) + { + if (_hardCancel.IsCancellationRequested) return; + + if (!reader.TryFillBatch(1024 * 1024, out var batch)) + { + await Task.Delay(100, hardCancel); + continue; + } + + await LogShipper.ShipBuffer(connection, apiKey, batch.Value.AsArraySegment(), SendFailureHandling.Retry); + + if (bookmark.TrySet(new BookmarkValue(batch.Value.ReaderHead.Chunk, + batch.Value.ReaderHead.CommitHead))) + { + reader.AdvanceTo(batch.Value.ReaderHead); + } + + batch.Value.Return(); + } + }, cancellationToken: hardCancel); + } + + public async Task WriteAsync(byte[] storage, Range range, CancellationToken cancellationToken) + { + var tcs = new TaskCompletionSource(); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _hardCancel); + + await _writer.WriteAsync(new ForwardingChannelEntry(storage[range], tcs), cts.Token); + await tcs.Task; + } + + public async Task StopAsync() + { + await _stop.CancelAsync(); + + _writer.Complete(); + await _writeWorker; + + await _readWorker; + } +} diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannelEntry.cs b/src/SeqCli/Forwarder/Channel/ForwardingChannelEntry.cs new file mode 100644 index 00000000..3f7d5fa9 --- /dev/null +++ b/src/SeqCli/Forwarder/Channel/ForwardingChannelEntry.cs @@ -0,0 +1,6 @@ +using System; +using System.Threading.Tasks; + +namespace SeqCli.Forwarder.Channel; + +public readonly record struct ForwardingChannelEntry(ArraySegment Data, TaskCompletionSource CompletionSource); diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs b/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs new file mode 100644 index 00000000..100ca72e --- /dev/null +++ b/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Seq.Api; +using SeqCli.Forwarder.Filesystem.System; +using SeqCli.Forwarder.Storage; +using Serilog; + +namespace SeqCli.Forwarder.Channel; + +class ForwardingChannelMap +{ + readonly string _bufferPath; + readonly SeqConnection _connection; + readonly ForwardingChannel _defaultChannel; + readonly Lock _channelsSync = new(); + readonly Dictionary _channels = new(); + readonly CancellationTokenSource _shutdownTokenSource = new(); + + public ForwardingChannelMap(string bufferPath, SeqConnection connection, string? defaultApiKey) + { + _bufferPath = bufferPath; + _connection = connection; + _defaultChannel = OpenOrCreateChannel(defaultApiKey, "Default"); + + // TODO, load other channels at start-up + } + + ForwardingChannel OpenOrCreateChannel(string? apiKey, string name) + { + // TODO, when it's not the default, persist the API key and validate equality on reopen + + var storePath = Path.Combine(_bufferPath, name); + var defaultStore = new SystemStoreDirectory(storePath); + Log.Information("Opening local buffer in {StorePath}", storePath); + + return new ForwardingChannel( + BufferAppender.Open(defaultStore), + BufferReader.Open(defaultStore), + Bookmark.Open(defaultStore), + _connection, + apiKey, + _shutdownTokenSource.Token); + } + + public ForwardingChannel Get(string? apiKey) + { + if (string.IsNullOrWhiteSpace(apiKey)) + { + return _defaultChannel; + } + + lock (_channelsSync) + { + if (_channels.TryGetValue(apiKey, out var channel)) + { + return channel; + } + + // Seq API keys begin with four identifying characters that aren't considered part of the + // confidential key. TODO: we could likely do better than this. + var name = apiKey[..4]; + var created = OpenOrCreateChannel(apiKey, name); + _channels.Add(apiKey, created); + return created; + } + } + + public async Task StopAsync() + { + Log.Information("Flushing log buffers"); + + _shutdownTokenSource.CancelAfter(TimeSpan.FromSeconds(30)); + + Task[] stopChannels; + lock (_channelsSync) + { + stopChannels = _channels.Values.Select(ch => ch.StopAsync()).ToArray(); + } + + await Task.WhenAll([ + _defaultChannel.StopAsync(), + ..stopChannels]); + + await _shutdownTokenSource.CancelAsync(); + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Channel/LogChannel.cs b/src/SeqCli/Forwarder/Channel/LogChannel.cs deleted file mode 100644 index c1567813..00000000 --- a/src/SeqCli/Forwarder/Channel/LogChannel.cs +++ /dev/null @@ -1,57 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Channels; -using System.Threading.Tasks; - -namespace SeqCli.Forwarder.Channel; - -class LogChannel -{ - public LogChannel(Func, CancellationToken, Task> write) - { - var channel = System.Threading.Channels.Channel.CreateBounded(new BoundedChannelOptions(5) - { - SingleReader = false, - SingleWriter = true, - FullMode = BoundedChannelFullMode.Wait, - }); - - _shutdownTokenSource = new CancellationTokenSource(); - _writer = channel.Writer; - _worker = Task.Run(async () => - { - await foreach (var entry in channel.Reader.ReadAllAsync(_shutdownTokenSource.Token)) - { - try - { - await write(entry.Storage[entry.Range], _shutdownTokenSource.Token); - entry.Completion.SetResult(); - } - catch (Exception e) - { - entry.Completion.TrySetException(e); - } - } - }, cancellationToken: _shutdownTokenSource.Token); - } - - readonly ChannelWriter _writer; - readonly Task _worker; - readonly CancellationTokenSource _shutdownTokenSource; - - public async Task WriteAsync(byte[] storage, Range range, CancellationToken cancellationToken) - { - var tcs = new TaskCompletionSource(); - using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownTokenSource.Token); - - await _writer.WriteAsync(new LogChannelEntry(storage, range, tcs), cts.Token); - await tcs.Task; - } - - public async Task StopAsync() - { - _writer.Complete(); - await _worker; - await _shutdownTokenSource.CancelAsync(); - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Channel/LogChannelEntry.cs b/src/SeqCli/Forwarder/Channel/LogChannelEntry.cs deleted file mode 100644 index 866d135a..00000000 --- a/src/SeqCli/Forwarder/Channel/LogChannelEntry.cs +++ /dev/null @@ -1,6 +0,0 @@ -using System; -using System.Threading.Tasks; - -namespace SeqCli.Forwarder.Channel; - -public readonly record struct LogChannelEntry(byte[] Storage, Range Range, TaskCompletionSource Completion); diff --git a/src/SeqCli/Forwarder/Channel/LogChannelMap.cs b/src/SeqCli/Forwarder/Channel/LogChannelMap.cs deleted file mode 100644 index df7c3dd9..00000000 --- a/src/SeqCli/Forwarder/Channel/LogChannelMap.cs +++ /dev/null @@ -1,41 +0,0 @@ -using System; -using System.IO; -using System.Threading.Tasks; -using SeqCli.Forwarder.Filesystem.System; -using SeqCli.Forwarder.Storage; -using Serilog; - -namespace SeqCli.Forwarder.Channel; - -class LogChannelMap -{ - readonly BufferAppender _defaultAppender; - readonly LogChannel _defaultChannel; - - public LogChannelMap(string bufferPath) - { - var defaultStore = new SystemStoreDirectory(bufferPath); - Log.Information("Opening local buffer in {BufferPath}", bufferPath); - - _defaultAppender = BufferAppender.Open(defaultStore); - _defaultChannel = new LogChannel((chunk, _) => - { - // TODO: chunk sizes, max chunks, ingestion log - _defaultAppender.TryAppend(chunk.AsSpan(), 100_000_000); - return Task.CompletedTask; - }); - } - - public LogChannel Get(string? apiKey) - { - // apiKey is ignored. - return _defaultChannel; - } - - public async Task StopAsync() - { - Log.Information("Flushing log buffers"); - await _defaultChannel.StopAsync(); - _defaultAppender.Dispose(); - } -} diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 72f76e64..f06280c4 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -16,6 +16,7 @@ using System.Net.Http; using System.Threading; using Autofac; +using Seq.Api; using SeqCli.Config; using SeqCli.Forwarder.Channel; using SeqCli.Forwarder.Web.Api; @@ -28,17 +29,21 @@ class ForwarderModule : Module { readonly string _bufferPath; readonly SeqCliConfig _config; + readonly SeqConnection _connection; + readonly string? _apiKey; - public ForwarderModule(string bufferPath, SeqCliConfig config) + public ForwarderModule(string bufferPath, SeqCliConfig config, SeqConnection connection, string? apiKey) { _bufferPath = bufferPath ?? throw new ArgumentNullException(nameof(bufferPath)); _config = config ?? throw new ArgumentNullException(nameof(config)); + _connection = connection; + _apiKey = apiKey; } protected override void Load(ContainerBuilder builder) { builder.RegisterType().SingleInstance(); - builder.Register(_ => new LogChannelMap(_bufferPath)).SingleInstance(); + builder.Register(_ => new ForwardingChannelMap(_bufferPath, _connection, _apiKey)).SingleInstance(); builder.RegisterType().As(); builder.RegisterType().As(); diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs b/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs index 697b98e4..34492a0b 100644 --- a/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs +++ b/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs @@ -43,6 +43,11 @@ public ReadOnlySpan AsSpan() return _storage.AsSpan()[.._length]; } + public ArraySegment AsArraySegment() + { + return _storage[.._length]; + } + public void Return() { _pool?.Return(_storage); diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index e75655a7..00a44555 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -23,6 +23,7 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.Net.Http.Headers; +using SeqCli.Api; using SeqCli.Forwarder.Channel; using SeqCli.Forwarder.Diagnostics; using JsonException = System.Text.Json.JsonException; @@ -35,11 +36,11 @@ class IngestionEndpoints : IMapEndpoints { static readonly Encoding Utf8 = new UTF8Encoding(false); - readonly LogChannelMap _logChannels; + readonly ForwardingChannelMap _forwardingChannels; - public IngestionEndpoints(LogChannelMap logChannels) + public IngestionEndpoints(ForwardingChannelMap forwardingChannels) { - _logChannels = logChannels; + _forwardingChannels = forwardingChannels; } public void MapEndpoints(WebApplication app) @@ -84,7 +85,7 @@ static bool DefaultedBoolQuery(HttpRequest request, string queryParameterName) static string? GetApiKey(HttpRequest request) { - var apiKeyHeader = request.Headers["X-SeqApiKey"]; + var apiKeyHeader = request.Headers[ApiConstants.ApiKeyHeaderName]; if (apiKeyHeader.Count > 0) return apiKeyHeader.Last(); return request.Query.TryGetValue("apiKey", out var apiKey) ? apiKey.Last() : null; @@ -95,7 +96,7 @@ async Task IngestCompactFormatAsync(HttpContext context) var cts = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted); cts.CancelAfter(TimeSpan.FromSeconds(5)); - var log = _logChannels.Get(GetApiKey(context.Request)); + var log = _forwardingChannels.Get(GetApiKey(context.Request)); var payload = ArrayPool.Shared.Rent(1024 * 1024 * 10); var writeHead = 0; @@ -225,11 +226,11 @@ static bool ValidateClef(Span evt) return true; } - static async Task Write(LogChannel log, ArrayPool pool, byte[] storage, Range range, CancellationToken cancellationToken) + static async Task Write(ForwardingChannel forwardingChannel, ArrayPool pool, byte[] storage, Range range, CancellationToken cancellationToken) { try { - await log.WriteAsync(storage, range, cancellationToken); + await forwardingChannel.WriteAsync(storage, range, cancellationToken); } catch { diff --git a/src/SeqCli/Forwarder/Web/Host/ServerService.cs b/src/SeqCli/Forwarder/Web/Host/ServerService.cs index a8fb7e75..ca6545e9 100644 --- a/src/SeqCli/Forwarder/Web/Host/ServerService.cs +++ b/src/SeqCli/Forwarder/Web/Host/ServerService.cs @@ -24,13 +24,13 @@ namespace SeqCli.Forwarder.Web.Host; class ServerService { readonly IHost _host; - readonly LogChannelMap _logChannelMap; + readonly ForwardingChannelMap _forwardingChannelMap; readonly string _listenUri; - public ServerService(IHost host, LogChannelMap logChannelMap, string listenUri) + public ServerService(IHost host, ForwardingChannelMap forwardingChannelMap, string listenUri) { _host = host; - _logChannelMap = logChannelMap; + _forwardingChannelMap = forwardingChannelMap; _listenUri = listenUri; } @@ -60,6 +60,6 @@ public async Task StopAsync() Log.Information("HTTP server stopped; flushing buffers..."); - await _logChannelMap.StopAsync(); + await _forwardingChannelMap.StopAsync(); } } \ No newline at end of file diff --git a/src/SeqCli/Ingestion/LogShipper.cs b/src/SeqCli/Ingestion/LogShipper.cs index faeffe16..94f15298 100644 --- a/src/SeqCli/Ingestion/LogShipper.cs +++ b/src/SeqCli/Ingestion/LogShipper.cs @@ -16,6 +16,7 @@ using System.Collections.Generic; using System.IO; using System.Net.Http; +using System.Net.Http.Headers; using System.Text; using System.Threading.Tasks; using Newtonsoft.Json; @@ -31,6 +32,56 @@ namespace SeqCli.Ingestion; static class LogShipper { static readonly ITextFormatter JsonFormatter = OutputFormatter.Json(null); + + public static async Task ShipBuffer( + SeqConnection connection, + string? apiKey, + ArraySegment utf8Clef, + SendFailureHandling sendFailureHandling) + { + var content = new ByteArrayContent(utf8Clef.Array!, utf8Clef.Offset, utf8Clef.Count) + { + Headers = + { + ContentType = new MediaTypeHeaderValue(ApiConstants.ClefMediaType, "utf-8") + } + }; + + var retries = 0; + while (true) + { + var sendSucceeded = false; + try + { + sendSucceeded = await Send( + connection, + apiKey, + sendFailureHandling != SendFailureHandling.Ignore, + content); + } + catch (Exception ex) + { + if (sendFailureHandling != SendFailureHandling.Ignore) + Log.Error(ex, "Failed to send an event batch"); + } + + if (!sendSucceeded) + { + if (sendFailureHandling == SendFailureHandling.Fail) + return false; + + if (sendFailureHandling == SendFailureHandling.Retry) + { + var millisecondsDelay = (int)Math.Min(Math.Pow(2, retries) * 2000, 60000); + await Task.Delay(millisecondsDelay); + retries += 1; + continue; + } + } + + return true; + } + } public static async Task ShipEvents( SeqConnection connection, @@ -163,6 +214,11 @@ static async Task SendBatchAsync( content = new StringContent(builder.ToString(), Encoding.UTF8, ApiConstants.ClefMediaType); } + return await Send(connection, apiKey, logSendFailures, content); + } + + static async Task Send(SeqConnection connection, string? apiKey, bool logSendFailures, HttpContent content) + { var request = new HttpRequestMessage(HttpMethod.Post, ApiConstants.IngestionEndpoint) { Content = content }; if (apiKey != null) request.Headers.Add(ApiConstants.ApiKeyHeaderName, apiKey); diff --git a/src/SeqCli/Program.cs b/src/SeqCli/Program.cs index 4c58631f..9b4f8ffe 100644 --- a/src/SeqCli/Program.cs +++ b/src/SeqCli/Program.cs @@ -17,6 +17,7 @@ using System.Threading.Tasks; using Autofac; using SeqCli.Cli; +using SeqCli.Cli.Features; using SeqCli.Util; using Serilog; using Serilog.Core; diff --git a/src/SeqCli/SeqCliModule.cs b/src/SeqCli/SeqCliModule.cs index 1e085d67..2b04d6f7 100644 --- a/src/SeqCli/SeqCliModule.cs +++ b/src/SeqCli/SeqCliModule.cs @@ -15,6 +15,7 @@ using System.Reflection; using Autofac; using SeqCli.Cli; +using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Connection; using SeqCli.Encryptor; @@ -25,12 +26,13 @@ class SeqCliModule : Autofac.Module { protected override void Load(ContainerBuilder builder) { + builder.RegisterInstance(new StoragePathFeature()); builder.RegisterType(); builder.RegisterAssemblyTypes(typeof(Program).GetTypeInfo().Assembly) .As() .WithMetadataFrom(); builder.RegisterType(); - builder.Register(c => RuntimeConfigurationLoader.Load()).SingleInstance(); + builder.Register(c => RuntimeConfigurationLoader.Load(c.Resolve())).SingleInstance(); builder.Register(c => c.Resolve().Connection).SingleInstance(); builder.Register(c => c.Resolve().Output).SingleInstance(); builder.Register(c => c.Resolve().Encryption.DataProtector()).As(); diff --git a/test/SeqCli.Tests/Cli/CommandLineHostTests.cs b/test/SeqCli.Tests/Cli/CommandLineHostTests.cs index ade3c393..377dff0c 100644 --- a/test/SeqCli.Tests/Cli/CommandLineHostTests.cs +++ b/test/SeqCli.Tests/Cli/CommandLineHostTests.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Autofac.Features.Metadata; using SeqCli.Cli; +using SeqCli.Cli.Features; using SeqCli.Tests.Support; using Serilog.Core; using Serilog.Events; @@ -26,7 +27,7 @@ public async Task CommandLineHostPicksCorrectCommand() new Lazy(() => new ActionCommand(() => executed.Add("test2"))), new CommandMetadata {Name = "test2", HelpText = "help"}) }; - var commandLineHost = new CommandLineHost(availableCommands); + var commandLineHost = new CommandLineHost(availableCommands, new StoragePathFeature()); await commandLineHost.Run(["test"],new LoggingLevelSwitch()); Assert.Equal("test", executed.Single()); @@ -42,7 +43,7 @@ public async Task PrereleaseCommandsAreIgnoredWithoutFlag() new Lazy(() => new ActionCommand(() => executed.Add("test"))), new CommandMetadata {Name = "test", HelpText = "help", IsPreview = true}), }; - var commandLineHost = new CommandLineHost(availableCommands); + var commandLineHost = new CommandLineHost(availableCommands, new StoragePathFeature()); var exit = await commandLineHost.Run(["test"],new LoggingLevelSwitch()); Assert.Equal(1, exit); Assert.Empty(executed); @@ -66,7 +67,7 @@ public async Task WhenMoreThanOneSubcommandAndTheUserRunsWithSubcommandEnsurePic new Lazy(() => new ActionCommand(() => executed.Add("test-subcommand2"))), new CommandMetadata {Name = "test", SubCommand = "subcommand2", HelpText = "help"}) }; - var commandLineHost = new CommandLineHost(availableCommands); + var commandLineHost = new CommandLineHost(availableCommands, new StoragePathFeature()); await commandLineHost.Run(["test", "subcommand2"], new LoggingLevelSwitch()); Assert.Equal("test-subcommand2", executed.First()); @@ -85,7 +86,7 @@ public async Task VerboseOptionSetsLoggingLevelToInformation() new CommandMetadata {Name = "test", HelpText = "help"}) }; - var commandLineHost = new CommandLineHost(availableCommands); + var commandLineHost = new CommandLineHost(availableCommands, new StoragePathFeature()); await commandLineHost.Run(["test", "--verbose"], levelSwitch); From 1c44383cd514528fedc0065f3f87f7cbe4871b76 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 18 Jul 2025 13:57:52 +1000 Subject: [PATCH 40/51] Remove .DS_Store --- .DS_Store | Bin 6148 -> 0 bytes .gitignore | 2 ++ 2 files changed, 2 insertions(+) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 1a8b21bcb239f7d0ec1a07dcfc24e16d43c2a3a8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK-AcnS6i&A3I)<rAAeI4GAHmGT z{yN~d*IC9=_7D``zdw$$+_pb?t=`z$Zdy&NZQXfKvhZ?0pJyXKy+P|z$~dTWKe&pf z#iX-)CX?Kcl4+_6qHqc+x7Se;%3>tvNtmfxPY0}))tYn;7K@|4GwgMRWA~k9S1g7- zyDJ7KgXOYi?HwMUUQC|jmqfm4MmZ3!WXE6y@1U3ky?SX9%j6L}Rd$s{NDL4I!~ij{ z!3>x~!D??X2efozfEZZM0PYV08lr2k)Tp)&=7p`UpzfkFnyBet{28e-G2I|`E;Q4 Date: Fri, 18 Jul 2025 14:08:37 +1000 Subject: [PATCH 41/51] Fix help --pre --- src/SeqCli/Cli/CommandLineHost.cs | 2 +- src/SeqCli/Cli/Commands/HelpCommand.cs | 2 ++ src/SeqCli/Properties/launchSettings.json | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/SeqCli/Cli/CommandLineHost.cs b/src/SeqCli/Cli/CommandLineHost.cs index 0c156456..23e45850 100644 --- a/src/SeqCli/Cli/CommandLineHost.cs +++ b/src/SeqCli/Cli/CommandLineHost.cs @@ -57,7 +57,7 @@ public async Task Run(string[] args, LoggingLevelSwitch levelSwitch) if (cmd != null) { var amountToSkip = cmd.Metadata.SubCommand == null ? 1 : 2; - var commandSpecificArgs = args.Skip(amountToSkip).Where(arg => arg != prereleaseArg).ToArray(); + var commandSpecificArgs = args.Skip(amountToSkip).Where(arg => cmd.Metadata.Name == "help" || arg != prereleaseArg).ToArray(); var verbose = commandSpecificArgs.Any(arg => arg == verboseArg); if (verbose) diff --git a/src/SeqCli/Cli/Commands/HelpCommand.cs b/src/SeqCli/Cli/Commands/HelpCommand.cs index 3aab532b..57b7d64d 100644 --- a/src/SeqCli/Cli/Commands/HelpCommand.cs +++ b/src/SeqCli/Cli/Commands/HelpCommand.cs @@ -37,6 +37,8 @@ public HelpCommand(IEnumerable, CommandMetadata>> availableCo protected override Task Run(string[] unrecognized) { + Console.WriteLine(_pre); + var orderedCommands = _availableCommands .Where(c => !c.Metadata.IsPreview || _pre) .OrderBy(c => c.Metadata.Name) diff --git a/src/SeqCli/Properties/launchSettings.json b/src/SeqCli/Properties/launchSettings.json index f9a7cad5..11a965f9 100644 --- a/src/SeqCli/Properties/launchSettings.json +++ b/src/SeqCli/Properties/launchSettings.json @@ -3,7 +3,7 @@ "profiles": { "SeqCli": { "commandName": "Project", - "commandLineArgs": "forwarder run --pre" + "commandLineArgs": "help --pre" } } } From 3e12c359038cdf69c604707c3f16719168e09ca4 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 18 Jul 2025 15:27:48 +1000 Subject: [PATCH 42/51] Don't inject SeqCliConfig into CLI commands --- src/SeqCli/Cli/CommandLineHost.cs | 5 -- .../Cli/Commands/ApiKey/CreateCommand.cs | 22 ++++-- src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs | 12 +-- .../Cli/Commands/ApiKey/RemoveCommand.cs | 10 ++- src/SeqCli/Cli/Commands/App/DefineCommand.cs | 1 + src/SeqCli/Cli/Commands/App/InstallCommand.cs | 13 +-- src/SeqCli/Cli/Commands/App/ListCommand.cs | 20 ++--- src/SeqCli/Cli/Commands/App/RunCommand.cs | 9 ++- .../Cli/Commands/App/UninstallCommand.cs | 8 +- src/SeqCli/Cli/Commands/App/UpdateCommand.cs | 16 ++-- .../Cli/Commands/AppInstance/CreateCommand.cs | 13 +-- .../Cli/Commands/AppInstance/ListCommand.cs | 18 +++-- .../Cli/Commands/AppInstance/RemoveCommand.cs | 8 +- src/SeqCli/Cli/Commands/Bench/BenchCommand.cs | 23 +++--- .../Cli/Commands/Cluster/HealthCommand.cs | 23 +++--- src/SeqCli/Cli/Commands/ConfigCommand.cs | 4 +- .../Cli/Commands/Dashboard/ListCommand.cs | 18 +++-- .../Cli/Commands/Dashboard/RemoveCommand.cs | 8 +- .../Cli/Commands/Dashboard/RenderCommand.cs | 19 +++-- .../Commands/ExpressionIndex/CreateCommand.cs | 13 +-- .../Commands/ExpressionIndex/ListCommand.cs | 13 +-- .../Commands/ExpressionIndex/RemoveCommand.cs | 7 +- src/SeqCli/Cli/Commands/Feed/CreateCommand.cs | 13 +-- src/SeqCli/Cli/Commands/Feed/ListCommand.cs | 18 +++-- src/SeqCli/Cli/Commands/Feed/RemoveCommand.cs | 8 +- .../Cli/Commands/Forwarder/InstallCommand.cs | 15 ++-- .../Cli/Commands/Forwarder/RestartCommand.cs | 79 +++++++++---------- .../Cli/Commands/Forwarder/RunCommand.cs | 15 ++-- .../Cli/Commands/Forwarder/StartCommand.cs | 65 ++++++++------- .../Cli/Commands/Forwarder/StatusCommand.cs | 43 +++++----- .../Cli/Commands/Forwarder/StopCommand.cs | 67 ++++++++-------- .../Cli/Commands/Forwarder/TruncateCommand.cs | 4 +- .../Commands/Forwarder/UninstallCommand.cs | 37 +++++---- src/SeqCli/Cli/Commands/HelpCommand.cs | 2 - src/SeqCli/Cli/Commands/Index/ListCommand.cs | 13 +-- .../Cli/Commands/Index/SuppressCommand.cs | 9 ++- src/SeqCli/Cli/Commands/IngestCommand.cs | 8 +- .../Cli/Commands/License/ApplyCommand.cs | 8 +- src/SeqCli/Cli/Commands/LogCommand.cs | 8 +- src/SeqCli/Cli/Commands/Node/HealthCommand.cs | 27 ++++--- src/SeqCli/Cli/Commands/Node/ListCommand.cs | 17 ++-- src/SeqCli/Cli/Commands/PrintCommand.cs | 8 +- .../Cli/Commands/Profile/CreateCommand.cs | 6 +- .../Cli/Commands/Profile/ListCommand.cs | 6 +- .../Cli/Commands/Profile/RemoveCommand.cs | 4 +- src/SeqCli/Cli/Commands/QueryCommand.cs | 17 ++-- .../Commands/RetentionPolicy/CreateCommand.cs | 13 +-- .../Commands/RetentionPolicy/ListCommand.cs | 17 ++-- .../Commands/RetentionPolicy/RemoveCommand.cs | 8 +- .../Cli/Commands/Sample/IngestCommand.cs | 10 ++- .../Cli/Commands/Sample/SetupCommand.cs | 16 ++-- src/SeqCli/Cli/Commands/SearchCommand.cs | 11 ++- .../Cli/Commands/Settings/ClearCommand.cs | 8 +- .../Cli/Commands/Settings/SetCommand.cs | 8 +- .../Cli/Commands/Settings/ShowCommand.cs | 8 +- .../Cli/Commands/Shared/UpdateCommand.cs | 6 +- .../Cli/Commands/Signal/CreateCommand.cs | 13 +-- .../Cli/Commands/Signal/ImportCommand.cs | 10 ++- src/SeqCli/Cli/Commands/Signal/ListCommand.cs | 18 +++-- .../Cli/Commands/Signal/RemoveCommand.cs | 8 +- src/SeqCli/Cli/Commands/TailCommand.cs | 13 +-- .../Cli/Commands/Template/ExportCommand.cs | 8 +- .../Cli/Commands/Template/ImportCommand.cs | 8 +- src/SeqCli/Cli/Commands/User/CreateCommand.cs | 13 +-- src/SeqCli/Cli/Commands/User/ListCommand.cs | 18 +++-- src/SeqCli/Cli/Commands/User/RemoveCommand.cs | 8 +- .../Cli/Commands/Workspace/CreateCommand.cs | 13 +-- .../Cli/Commands/Workspace/ListCommand.cs | 18 +++-- .../Cli/Commands/Workspace/RemoveCommand.cs | 8 +- .../Cli/Features/OutputFormatFeature.cs | 54 +++++++------ src/SeqCli/Cli/Features/StoragePathFeature.cs | 4 +- src/SeqCli/Connection/SeqConnectionFactory.cs | 21 ++--- src/SeqCli/SeqCliModule.cs | 10 +-- 73 files changed, 641 insertions(+), 489 deletions(-) diff --git a/src/SeqCli/Cli/CommandLineHost.cs b/src/SeqCli/Cli/CommandLineHost.cs index 23e45850..f221468b 100644 --- a/src/SeqCli/Cli/CommandLineHost.cs +++ b/src/SeqCli/Cli/CommandLineHost.cs @@ -67,11 +67,6 @@ public async Task Run(string[] args, LoggingLevelSwitch levelSwitch) } var impl = cmd.Value.Value; - - // This one is global, because implicitly-created components in the container rely on the specified - // storage path. It's a convoluted data flow that we should rip out when possible. - _storagePathFeature.Enable(impl.Options); - return await impl.Invoke(commandSpecificArgs); } } diff --git a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs index 68be490e..8c5c39c9 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs @@ -37,12 +37,13 @@ class CreateCommand : Command readonly ConnectionFeature _connection; readonly PropertiesFeature _properties; readonly OutputFormatFeature _output; + readonly StoragePathFeature _storagePath; string? _title, _token, _filter, _level, _connectUsername, _connectPassword; string[]? _permissions; bool _useServerTimestamps, _connectPasswordStdin; - public CreateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public CreateCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -94,12 +95,15 @@ public CreateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config _ => _connectPasswordStdin = true); _connection = Enable(); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); } protected override async Task Run() { - var connection = await TryConnectAsync(); + var config = RuntimeConfigurationLoader.Load(_storagePath); + + var connection = await TryConnectAsync(config); if (connection == null) return 1; @@ -149,19 +153,21 @@ protected override async Task Run() apiKey = await connection.ApiKeys.AddAsync(apiKey); - if (_token == null && !_output.Json) + var output = _output.GetOutputFormat(config); + + if (_token == null && !output.Json) { Console.WriteLine(apiKey.Token); } else { - _output.WriteEntity(apiKey); + output.WriteEntity(apiKey); } return 0; } - async Task TryConnectAsync() + async Task TryConnectAsync(SeqCliConfig config) { SeqConnection connection; if (_connectUsername != null) @@ -183,13 +189,13 @@ protected override async Task Run() _connectPassword = await Console.In.ReadLineAsync(); } - var (url, _) = _connectionFactory.GetConnectionDetails(_connection); + var (url, _) = _connectionFactory.GetConnectionDetails(_connection, config); connection = new SeqConnection(url); await connection.Users.LoginAsync(_connectUsername, _connectPassword ?? ""); } else { - connection = _connectionFactory.Connect(_connection); + connection = _connectionFactory.Connect(_connection, config); } return connection; diff --git a/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs index 5f07cb28..d206fd07 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs @@ -29,27 +29,29 @@ class ListCommand : Command readonly EntityIdentityFeature _entityIdentity; readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; + readonly StoragePathFeature _storagePath; - public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public ListCommand(SeqConnectionFactory connectionFactory) { - if (config == null) throw new ArgumentNullException(nameof(config)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); _entityIdentity = Enable(new EntityIdentityFeature("API key", "list")); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); _connection = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var list = _entityIdentity.Id != null ? new[] { await connection.ApiKeys.FindAsync(_entityIdentity.Id) } : (await connection.ApiKeys.ListAsync()) .Where(ak => _entityIdentity.Title == null || _entityIdentity.Title == ak.Title); - _output.ListEntities(list); + _output.GetOutputFormat(config).ListEntities(list); return 0; } diff --git a/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs index 2cac207d..357cbcaf 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs @@ -16,6 +16,7 @@ using System.Linq; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using Serilog; @@ -29,6 +30,7 @@ class RemoveCommand : Command readonly EntityIdentityFeature _entityIdentity; readonly ConnectionFeature _connection; + readonly StoragePathFeature _storagePath; public RemoveCommand(SeqConnectionFactory connectionFactory) { @@ -36,6 +38,7 @@ public RemoveCommand(SeqConnectionFactory connectionFactory) _entityIdentity = Enable(new EntityIdentityFeature("API key", "remove")); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -46,10 +49,11 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); - var toRemove = _entityIdentity.Id != null ? - new[] {await connection.ApiKeys.FindAsync(_entityIdentity.Id)} : + var toRemove = _entityIdentity.Id != null ? [await connection.ApiKeys.FindAsync(_entityIdentity.Id)] + : (await connection.ApiKeys.ListAsync()) .Where(ak => _entityIdentity.Title == ak.Title) .ToArray(); diff --git a/src/SeqCli/Cli/Commands/App/DefineCommand.cs b/src/SeqCli/Cli/Commands/App/DefineCommand.cs index 16214070..56909bbb 100644 --- a/src/SeqCli/Cli/Commands/App/DefineCommand.cs +++ b/src/SeqCli/Cli/Commands/App/DefineCommand.cs @@ -16,6 +16,7 @@ using System.Threading.Tasks; using SeqCli.Apps; using SeqCli.Apps.Definitions; +using SeqCli.Cli.Features; using SeqCli.Util; namespace SeqCli.Cli.Commands.App; diff --git a/src/SeqCli/Cli/Commands/App/InstallCommand.cs b/src/SeqCli/Cli/Commands/App/InstallCommand.cs index 7f9974d8..76a57cf7 100644 --- a/src/SeqCli/Cli/Commands/App/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/App/InstallCommand.cs @@ -33,10 +33,11 @@ class InstallCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; - + readonly StoragePathFeature _storagePath; + string? _packageId, _version, _feedId; - public InstallCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public InstallCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -56,7 +57,8 @@ public InstallCommand(SeqConnectionFactory connectionFactory, SeqCliConfig confi feedId => _feedId = ArgumentString.Normalize(feedId)); _connection = Enable(); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -67,7 +69,8 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var feedId = _feedId; if (feedId == null) @@ -83,7 +86,7 @@ protected override async Task Run() } var app = await connection.Apps.InstallPackageAsync(feedId, _packageId, _version); - _output.WriteEntity(app); + _output.GetOutputFormat(config).WriteEntity(app); return 0; } diff --git a/src/SeqCli/Cli/Commands/App/ListCommand.cs b/src/SeqCli/Cli/Commands/App/ListCommand.cs index 3bd47736..d0b1b41f 100644 --- a/src/SeqCli/Cli/Commands/App/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/App/ListCommand.cs @@ -15,13 +15,13 @@ class ListCommand : Command string? _title, _id; readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; - + readonly StoragePathFeature _storagePath; + string? PackageId => string.IsNullOrWhiteSpace(_title) ? null : _title.Trim(); string? Id => string.IsNullOrWhiteSpace(_id) ? null : _id.Trim(); - public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public ListCommand(SeqConnectionFactory connectionFactory) { - if (config == null) throw new ArgumentNullException(nameof(config)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); Options.Add( @@ -34,7 +34,8 @@ public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) "The id of a single app to list", t => _id = t); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); _connection = Enable(); } @@ -42,18 +43,19 @@ protected override async Task Run() { if (PackageId != null && Id != null) { - ShowUsageErrors(new[] {"Only one of either `package-id` or `id` can be specified"}); + ShowUsageErrors(["Only one of either `package-id` or `id` can be specified"]); return 1; } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); - var list = Id != null ? - new[] { await connection.Apps.FindAsync(Id) } : + var list = Id != null ? [await connection.Apps.FindAsync(Id)] + : (await connection.Apps.ListAsync()) .Where(ak => PackageId == null || PackageId == ak.Package.PackageId); - _output.ListEntities(list); + _output.GetOutputFormat(config).ListEntities(list); return 0; } diff --git a/src/SeqCli/Cli/Commands/App/RunCommand.cs b/src/SeqCli/Cli/Commands/App/RunCommand.cs index a680fd67..a3590f46 100644 --- a/src/SeqCli/Cli/Commands/App/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/App/RunCommand.cs @@ -16,6 +16,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using SeqCli.Apps.Hosting; +using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Util; @@ -39,10 +40,10 @@ class RunCommand : Command readonly Dictionary _settings = new(); - public RunCommand(SeqCliConfig config) + public RunCommand() { - if (config == null) throw new ArgumentNullException(nameof(config)); - _serverUrl = config.Connection.ServerUrl; + // The usual `--storage` argument is not supported on this command (see notes on `--storage` arg below). + _serverUrl = RuntimeConfigurationLoader.Load(new StoragePathFeature()).Connection.ServerUrl; Options.Add( "d=|directory=", @@ -64,6 +65,8 @@ public RunCommand(SeqCliConfig config) _settings.Add(name, valueText ?? ""); }); + // Important note, this conflicts with the `--storage` argument accepted by the majority of other commands; changing + // this requires an update to Seq, which uses this command for hosting .NET apps. Options.Add( "storage=", "A directory in which app-specific data can be stored; defaults to the current directory", diff --git a/src/SeqCli/Cli/Commands/App/UninstallCommand.cs b/src/SeqCli/Cli/Commands/App/UninstallCommand.cs index c9d275a9..5423c0ac 100644 --- a/src/SeqCli/Cli/Commands/App/UninstallCommand.cs +++ b/src/SeqCli/Cli/Commands/App/UninstallCommand.cs @@ -2,6 +2,7 @@ using System.Linq; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using SeqCli.Util; using Serilog; @@ -17,7 +18,8 @@ class UninstallCommand : Command string? _packageId, _id; readonly ConnectionFeature _connection; - + readonly StoragePathFeature _storagePath; + public UninstallCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -33,6 +35,7 @@ public UninstallCommand(SeqConnectionFactory connectionFactory) t => _id = ArgumentString.Normalize(t)); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -43,7 +46,8 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var toRemove = _id != null ? [await connection.Apps.FindAsync(_id)] : (await connection.Apps.ListAsync()) diff --git a/src/SeqCli/Cli/Commands/App/UpdateCommand.cs b/src/SeqCli/Cli/Commands/App/UpdateCommand.cs index e0051a9c..429c1977 100644 --- a/src/SeqCli/Cli/Commands/App/UpdateCommand.cs +++ b/src/SeqCli/Cli/Commands/App/UpdateCommand.cs @@ -32,11 +32,12 @@ class UpdateCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; - + readonly StoragePathFeature _storagePath; + string? _id, _name, _version; bool _all, _force; - public UpdateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public UpdateCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -67,7 +68,8 @@ public UpdateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config _ => _force = true); _connection = Enable(); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -90,8 +92,10 @@ protected override async Task Run() Log.Error("One of `id`, `name`, or `all` must be specified"); return 1; } - - var connection = _connectionFactory.Connect(_connection); + + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); + var output = _output.GetOutputFormat(config); var apps = await connection.Apps.ListAsync(); foreach (var app in apps) @@ -99,7 +103,7 @@ protected override async Task Run() if (_all || app.Id == _id || _name != null && _name.Equals(app.Name, StringComparison.OrdinalIgnoreCase)) { var updated = await connection.Apps.UpdatePackageAsync(app, _version, _force); - _output.WriteEntity(updated); + output.WriteEntity(updated); } } diff --git a/src/SeqCli/Cli/Commands/AppInstance/CreateCommand.cs b/src/SeqCli/Cli/Commands/AppInstance/CreateCommand.cs index 5b71c2cc..56b2e03d 100644 --- a/src/SeqCli/Cli/Commands/AppInstance/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/AppInstance/CreateCommand.cs @@ -20,13 +20,14 @@ class CreateCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; - + readonly StoragePathFeature _storagePath; + string? _title, _appId, _streamIncomingEventsSignal; readonly Dictionary _settings = new(); readonly List _overridable = new(); bool _streamIncomingEvents; - public CreateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public CreateCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -70,12 +71,14 @@ public CreateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config s => _overridable.Add(s)); _connection = Enable(); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); AppInstanceEntity instance = await connection.AppInstances.TemplateAsync(_appId)!; @@ -112,7 +115,7 @@ bool ValidateSettingName(string settingName) instance = await connection.AppInstances.AddAsync(instance); - _output.WriteEntity(instance); + _output.GetOutputFormat(config).WriteEntity(instance); return 0; } diff --git a/src/SeqCli/Cli/Commands/AppInstance/ListCommand.cs b/src/SeqCli/Cli/Commands/AppInstance/ListCommand.cs index b9f9335e..2d67736d 100644 --- a/src/SeqCli/Cli/Commands/AppInstance/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/AppInstance/ListCommand.cs @@ -15,27 +15,29 @@ class ListCommand : Command readonly EntityIdentityFeature _entityIdentity; readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; - - public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + readonly StoragePathFeature _storagePath; + + public ListCommand(SeqConnectionFactory connectionFactory) { - if (config == null) throw new ArgumentNullException(nameof(config)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); _entityIdentity = Enable(new EntityIdentityFeature("app instance", "list")); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); _connection = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); - var list = _entityIdentity.Id != null ? - new[] { await connection.AppInstances.FindAsync(_entityIdentity.Id) } : + var list = _entityIdentity.Id != null ? [await connection.AppInstances.FindAsync(_entityIdentity.Id)] + : (await connection.AppInstances.ListAsync()) .Where(d => _entityIdentity.Title == null || _entityIdentity.Title == d.Title); - _output.ListEntities(list); + _output.GetOutputFormat(config).ListEntities(list); return 0; } diff --git a/src/SeqCli/Cli/Commands/AppInstance/RemoveCommand.cs b/src/SeqCli/Cli/Commands/AppInstance/RemoveCommand.cs index 376f7354..a87f0d72 100644 --- a/src/SeqCli/Cli/Commands/AppInstance/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/AppInstance/RemoveCommand.cs @@ -2,6 +2,7 @@ using System.Linq; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using Serilog; @@ -15,13 +16,15 @@ class RemoveCommand : Command readonly EntityIdentityFeature _entityIdentity; readonly ConnectionFeature _connection; - + readonly StoragePathFeature _storagePath; + public RemoveCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); _entityIdentity = Enable(new EntityIdentityFeature("app instance", "remove")); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -32,7 +35,8 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var toRemove = _entityIdentity.Id != null ? [await connection.AppInstances.FindAsync(_entityIdentity.Id)] : diff --git a/src/SeqCli/Cli/Commands/Bench/BenchCommand.cs b/src/SeqCli/Cli/Commands/Bench/BenchCommand.cs index 9e14f039..fab25a41 100644 --- a/src/SeqCli/Cli/Commands/Bench/BenchCommand.cs +++ b/src/SeqCli/Cli/Commands/Bench/BenchCommand.cs @@ -23,6 +23,7 @@ using Seq.Api.Model.Data; using Seq.Api.Model.Signals; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using SeqCli.Sample.Loader; using SeqCli.Util; @@ -70,6 +71,7 @@ class BenchCommand : Command readonly ConnectionFeature _connection; readonly DateRangeFeature _range; readonly TimeoutFeature _timeout; + readonly StoragePathFeature _storagePath; string _cases = ""; string _reportingServerUrl = ""; string _reportingServerApiKey = ""; @@ -111,6 +113,8 @@ public BenchCommand(SeqConnectionFactory connectionFactory) "with-queries", "Should the benchmark include querying Seq", _ => _withQueries = true); + + _storagePath = Enable(); } protected override async Task Run() @@ -123,8 +127,9 @@ protected override async Task Run() try { - var (_, apiKey) = _connectionFactory.GetConnectionDetails(_connection); - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var (_, apiKey) = _connectionFactory.GetConnectionDetails(_connection, config); + var connection = _connectionFactory.Connect(_connection, config); var timeout = _timeout.ApplyTimeout(connection.Client.HttpClient); var seqVersion = (await connection.Client.GetRootAsync()).Version; await using var reportingLogger = BuildReportingLogger(); @@ -159,9 +164,9 @@ protected override async Task Run() if (!_withQueries) { - int benchDurationMs = 120_000; - await Task.Delay(benchDurationMs); - cancellationTokenSource.Cancel(); + const int benchDurationMs = 120_000; + await Task.Delay(benchDurationMs, cancellationToken); + await cancellationTokenSource.CancelAsync(); var response = await connection.Data.QueryAsync( "select count(*) from stream group by time(1s)", @@ -199,7 +204,7 @@ protected override async Task Run() { var collectedTimings = await QueryBenchmark(reportingLogger, runId, connection, seqVersion, timeout); collectedTimings.LogSummary(_description); - cancellationTokenSource.Cancel(); + await cancellationTokenSource.CancelAsync(); } } @@ -212,7 +217,7 @@ protected override async Task Run() } } - async Task IngestionBenchmark(Logger reportingLogger, string runId, SeqConnection connection, string? apiKey, + static async Task IngestionBenchmark(Logger reportingLogger, string runId, SeqConnection connection, string? apiKey, string seqVersion, bool isQueryBench, CancellationToken cancellationToken = default) { reportingLogger.Information( @@ -224,7 +229,7 @@ async Task IngestionBenchmark(Logger reportingLogger, string runId, SeqConnectio var simulationTasks = Enumerable.Range(1, 500) .Select(i => Simulation.RunAsync(connection, apiKey, 10000, echoToStdout: false, cancellationToken)) .ToArray(); - await Task.Delay(20_000); // how long to ingest before beginning queries + await Task.Delay(20_000, cancellationToken); // how long to ingest before beginning queries } else { @@ -245,7 +250,7 @@ async Task QueryBenchmark(Logger reportingLogger, string r foreach (var c in cases.Cases.OrderBy(c => c.Id) - .Concat(new [] { QueryBenchRunResults.FINAL_COUNT_CASE })) + .Concat([QueryBenchRunResults.FINAL_COUNT_CASE])) { var timings = new QueryBenchCaseTimings(c); queryBenchRunResults.Add(timings); diff --git a/src/SeqCli/Cli/Commands/Cluster/HealthCommand.cs b/src/SeqCli/Cli/Commands/Cluster/HealthCommand.cs index 75906eca..e3daa609 100644 --- a/src/SeqCli/Cli/Commands/Cluster/HealthCommand.cs +++ b/src/SeqCli/Cli/Commands/Cluster/HealthCommand.cs @@ -37,32 +37,35 @@ class HealthCommand : Command readonly OutputFormatFeature _output; readonly TimeoutFeature _timeout; readonly WaitUntilHealthyFeature _waitUntilHealthy; + readonly StoragePathFeature _storagePath; - public HealthCommand(SeqConnectionFactory connectionFactory, SeqCliOutputConfig seqCliOutputConfig) + public HealthCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); _waitUntilHealthy = Enable(new WaitUntilHealthyFeature("cluster")); _timeout = Enable(new TimeoutFeature()); - _output = Enable(new OutputFormatFeature(seqCliOutputConfig)); + _output = Enable(); + _storagePath = Enable(); _connection = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var timeout = _timeout.ApplyTimeout(connection.Client.HttpClient); if (_waitUntilHealthy.ShouldWait) { - return await RunUntilHealthy(connection, timeout ?? TimeSpan.FromSeconds(30)); + return await RunUntilHealthy(connection, timeout ?? TimeSpan.FromSeconds(30), _output.GetOutputFormat(config)); } - return await RunOnce(connection); + return await RunOnce(connection, _output.GetOutputFormat(config)); } - async Task RunUntilHealthy(SeqConnection connection, TimeSpan timeout) + async Task RunUntilHealthy(SeqConnection connection, TimeSpan timeout, OutputFormat outputFormat) { using var ct = new CancellationTokenSource(timeout); @@ -78,7 +81,7 @@ async Task RunUntilHealthy(SeqConnection connection, TimeSpan timeout) { try { - if (await RunOnce(connection) == 0) + if (await RunOnce(connection, outputFormat) == 0) { return 0; } @@ -98,13 +101,13 @@ async Task RunUntilHealthy(SeqConnection connection, TimeSpan timeout) } } - async Task RunOnce(SeqConnection connection) + static async Task RunOnce(SeqConnection connection, OutputFormat output) { var health = await connection.Cluster.CheckHealthAsync(); - if (_output.Json) + if (output.Json) { - _output.WriteObject(health); + output.WriteObject(health); } else if (!string.IsNullOrWhiteSpace(health.Description)) { Console.WriteLine($"{health.Status}: {health.Description}"); } else { diff --git a/src/SeqCli/Cli/Commands/ConfigCommand.cs b/src/SeqCli/Cli/Commands/ConfigCommand.cs index 2e71f07e..08249429 100644 --- a/src/SeqCli/Cli/Commands/ConfigCommand.cs +++ b/src/SeqCli/Cli/Commands/ConfigCommand.cs @@ -28,12 +28,12 @@ class ConfigCommand : Command bool _clear; readonly StoragePathFeature _storagePath; - public ConfigCommand(StoragePathFeature storagePath) + public ConfigCommand() { Options.Add("k|key=", "The field, for example `connection.serverUrl`", k => _key = k); Options.Add("v|value=", "The field value; if not specified, the command will print the current value", v => _value = v); Options.Add("c|clear", "Clear the field", _ => _clear = true); - _storagePath = storagePath; + _storagePath = Enable(); } protected override Task Run() diff --git a/src/SeqCli/Cli/Commands/Dashboard/ListCommand.cs b/src/SeqCli/Cli/Commands/Dashboard/ListCommand.cs index 1325511c..357fc99c 100644 --- a/src/SeqCli/Cli/Commands/Dashboard/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Dashboard/ListCommand.cs @@ -30,28 +30,30 @@ class ListCommand : Command readonly EntityOwnerFeature _entityOwner; readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; - - public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + readonly StoragePathFeature _storagePath; + + public ListCommand(SeqConnectionFactory connectionFactory) { - if (config == null) throw new ArgumentNullException(nameof(config)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); _entityIdentity = Enable(new EntityIdentityFeature("dashboard", "list")); _entityOwner = Enable(new EntityOwnerFeature("dashboard", "list", "listed", _entityIdentity)); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); _connection = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); - var list = _entityIdentity.Id != null ? - new[] { await connection.Dashboards.FindAsync(_entityIdentity.Id) } : + var list = _entityIdentity.Id != null ? [await connection.Dashboards.FindAsync(_entityIdentity.Id)] + : (await connection.Dashboards.ListAsync(ownerId: _entityOwner.OwnerId, shared: _entityOwner.IncludeShared)) .Where(d => _entityIdentity.Title == null || _entityIdentity.Title == d.Title); - _output.ListEntities(list); + _output.GetOutputFormat(config).ListEntities(list); return 0; } diff --git a/src/SeqCli/Cli/Commands/Dashboard/RemoveCommand.cs b/src/SeqCli/Cli/Commands/Dashboard/RemoveCommand.cs index c75fe807..b039180c 100644 --- a/src/SeqCli/Cli/Commands/Dashboard/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/Dashboard/RemoveCommand.cs @@ -16,6 +16,7 @@ using System.Linq; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using Serilog; @@ -30,7 +31,8 @@ class RemoveCommand : Command readonly EntityIdentityFeature _entityIdentity; readonly EntityOwnerFeature _entityOwner; readonly ConnectionFeature _connection; - + readonly StoragePathFeature _storagePath; + public RemoveCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -38,6 +40,7 @@ public RemoveCommand(SeqConnectionFactory connectionFactory) _entityIdentity = Enable(new EntityIdentityFeature("dashboard", "remove")); _entityOwner = Enable(new EntityOwnerFeature("dashboard", "remove", "removed", _entityIdentity)); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -48,7 +51,8 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var toRemove = _entityIdentity.Id != null ? [await connection.Dashboards.FindAsync(_entityIdentity.Id)] : diff --git a/src/SeqCli/Cli/Commands/Dashboard/RenderCommand.cs b/src/SeqCli/Cli/Commands/Dashboard/RenderCommand.cs index 83c4150d..683efc0d 100644 --- a/src/SeqCli/Cli/Commands/Dashboard/RenderCommand.cs +++ b/src/SeqCli/Cli/Commands/Dashboard/RenderCommand.cs @@ -40,12 +40,12 @@ class RenderCommand : Command readonly OutputFormatFeature _output; readonly SignalExpressionFeature _signal; readonly TimeoutFeature _timeout; - + readonly StoragePathFeature _storagePath; + string? _id, _lastDuration, _intervalDuration, _chartTitle; - public RenderCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public RenderCommand(SeqConnectionFactory connectionFactory) { - if (config == null) throw new ArgumentNullException(nameof(config)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); Options.Add( @@ -63,13 +63,15 @@ public RenderCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config _range = Enable(); _signal = Enable(); _timeout = Enable(); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); _connection = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); if (_id == null) { @@ -158,8 +160,9 @@ protected override async Task Run() var q = BuildSqlQuery(query, rangeStart, rangeEnd, timeGrouping); var timeout = _timeout.ApplyTimeout(connection.Client.HttpClient); - - if (_output.Json) + + var output = _output.GetOutputFormat(config); + if (output.Json) { var result = await connection.Data.QueryAsync(q, signal: signal, timeout: timeout); @@ -169,7 +172,7 @@ protected override async Task Run() else { var result = await connection.Data.QueryCsvAsync(q, signal: signal, timeout: timeout); - _output.WriteCsv(result); + output.WriteCsv(result); } return 0; diff --git a/src/SeqCli/Cli/Commands/ExpressionIndex/CreateCommand.cs b/src/SeqCli/Cli/Commands/ExpressionIndex/CreateCommand.cs index 5fc39086..d2b0d912 100644 --- a/src/SeqCli/Cli/Commands/ExpressionIndex/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/ExpressionIndex/CreateCommand.cs @@ -33,10 +33,11 @@ class CreateCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; - + readonly StoragePathFeature _storagePath; + string? _expression; - public CreateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public CreateCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -46,12 +47,14 @@ public CreateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config v => _expression = ArgumentString.Normalize(v)); _connection = Enable(); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); if (string.IsNullOrEmpty(_expression)) { @@ -63,7 +66,7 @@ protected override async Task Run() index.Expression = _expression; index = await connection.ExpressionIndexes.AddAsync(index); - _output.WriteEntity(index); + _output.GetOutputFormat(config).WriteEntity(index); return 0; } diff --git a/src/SeqCli/Cli/Commands/ExpressionIndex/ListCommand.cs b/src/SeqCli/Cli/Commands/ExpressionIndex/ListCommand.cs index 6836d3f1..6a8656af 100644 --- a/src/SeqCli/Cli/Commands/ExpressionIndex/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/ExpressionIndex/ListCommand.cs @@ -13,11 +13,12 @@ class ListCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; + readonly StoragePathFeature _storagePath; + string? _id; - public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public ListCommand(SeqConnectionFactory connectionFactory) { - if (config == null) throw new ArgumentNullException(nameof(config)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); Options.Add( @@ -25,17 +26,19 @@ public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) "The id of a single expression index to list", id => _id = id); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); _connection = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var list = _id is not null ? [await connection.ExpressionIndexes.FindAsync(_id)] : await connection.ExpressionIndexes.ListAsync(); - _output.ListEntities(list); + _output.GetOutputFormat(config).ListEntities(list); return 0; } } \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/ExpressionIndex/RemoveCommand.cs b/src/SeqCli/Cli/Commands/ExpressionIndex/RemoveCommand.cs index c7ebee37..21abc415 100644 --- a/src/SeqCli/Cli/Commands/ExpressionIndex/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/ExpressionIndex/RemoveCommand.cs @@ -13,9 +13,9 @@ // limitations under the License. using System; -using System.Linq; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using Serilog; @@ -28,6 +28,7 @@ class RemoveCommand : Command readonly SeqConnectionFactory _connectionFactory; readonly ConnectionFeature _connection; + readonly StoragePathFeature _storagePath; string? _id; public RemoveCommand(SeqConnectionFactory connectionFactory) @@ -40,6 +41,7 @@ public RemoveCommand(SeqConnectionFactory connectionFactory) id => _id = id); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -50,7 +52,8 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var toRemove = await connection.ExpressionIndexes.FindAsync(_id); await connection.ExpressionIndexes.RemoveAsync(toRemove); diff --git a/src/SeqCli/Cli/Commands/Feed/CreateCommand.cs b/src/SeqCli/Cli/Commands/Feed/CreateCommand.cs index 28a891c0..e7750cac 100644 --- a/src/SeqCli/Cli/Commands/Feed/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Feed/CreateCommand.cs @@ -30,11 +30,12 @@ class CreateCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; - + readonly StoragePathFeature _storagePath; + string? _name, _location, _username, _password; bool _passwordStdin; - public CreateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public CreateCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -64,12 +65,14 @@ public CreateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config _ => _passwordStdin = true); _connection = Enable(); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var feed = await connection.Feeds.TemplateAsync(); feed.Name = _name; @@ -94,7 +97,7 @@ protected override async Task Run() feed = await connection.Feeds.AddAsync(feed); - _output.WriteEntity(feed); + _output.GetOutputFormat(config).WriteEntity(feed); return 0; } diff --git a/src/SeqCli/Cli/Commands/Feed/ListCommand.cs b/src/SeqCli/Cli/Commands/Feed/ListCommand.cs index 31e27116..42c06565 100644 --- a/src/SeqCli/Cli/Commands/Feed/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Feed/ListCommand.cs @@ -28,12 +28,12 @@ class ListCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; - + readonly StoragePathFeature _storagePath; + string? _name, _id; - public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public ListCommand(SeqConnectionFactory connectionFactory) { - if (config == null) throw new ArgumentNullException(nameof(config)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); Options.Add( @@ -46,20 +46,22 @@ public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) "The id of a single feed to list", id => _id = id); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); _connection = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); - var list = _id != null ? - new[] { await connection.Feeds.FindAsync(_id) } : + var list = _id != null ? [await connection.Feeds.FindAsync(_id)] + : (await connection.Feeds.ListAsync()) .Where(f => _name == null || _name == f.Name); - _output.ListEntities(list); + _output.GetOutputFormat(config).ListEntities(list); return 0; } diff --git a/src/SeqCli/Cli/Commands/Feed/RemoveCommand.cs b/src/SeqCli/Cli/Commands/Feed/RemoveCommand.cs index c706fe3e..097c6d48 100644 --- a/src/SeqCli/Cli/Commands/Feed/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/Feed/RemoveCommand.cs @@ -16,6 +16,7 @@ using System.Linq; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using Serilog; @@ -28,7 +29,8 @@ class RemoveCommand : Command readonly SeqConnectionFactory _connectionFactory; readonly ConnectionFeature _connection; - + readonly StoragePathFeature _storagePath; + string? _name, _id; public RemoveCommand(SeqConnectionFactory connectionFactory) @@ -46,6 +48,7 @@ public RemoveCommand(SeqConnectionFactory connectionFactory) id => _id = id); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -56,7 +59,8 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var toRemove = _id != null ? [await connection.Feeds.FindAsync(_id)] : diff --git a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs index 63c8a2ee..330057de 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs @@ -38,19 +38,19 @@ class InstallCommand : Command readonly StoragePathFeature _storagePath; readonly ServiceCredentialsFeature _serviceCredentials; readonly ListenUriFeature _listenUri; - + bool _setup; - public InstallCommand(StoragePathFeature storagePath) + public InstallCommand() { - _storagePath = storagePath; + _storagePath = Enable(); _listenUri = Enable(); _serviceCredentials = Enable(); Options.Add( "setup", "Install and start the service only if it does not exist; otherwise reconfigure the binary location", - v => _setup = true); + _ => _setup = true); } string ServiceUsername => _serviceCredentials.IsUsernameSpecified ? _serviceCredentials.Username : "NT AUTHORITY\\LocalService"; @@ -106,10 +106,7 @@ int Setup() Console.WriteLine("Service is installed; checking path and dependency configuration..."); Reconfigure(controller); - if (controller.Status != ServiceControllerStatus.Running) - return Start(controller); - - return 0; + return controller.Status != ServiceControllerStatus.Running ? Start(controller) : 0; } static void Reconfigure(ServiceController controller) @@ -121,7 +118,7 @@ static void Reconfigure(ServiceController controller) if (!ServiceConfiguration.GetServiceBinaryPath(controller, out var path)) return; - var current = "\"" + Path.Combine(AppDomain.CurrentDomain.BaseDirectory!, Program.BinaryName) + "\""; + var current = "\"" + Path.Combine(AppDomain.CurrentDomain.BaseDirectory, Program.BinaryName) + "\""; if (path.StartsWith(current)) return; diff --git a/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs index e00842a6..ef028c90 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs @@ -22,62 +22,61 @@ // ReSharper disable UnusedType.Global -namespace SeqCli.Cli.Commands.Forwarder +namespace SeqCli.Cli.Commands.Forwarder; + +[Command("forwarder", "restart", "Restart the forwarder Windows service", IsPreview = true)] +[SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] +class RestartCommand : Command { - [Command("forwarder", "restart", "Restart the forwarder Windows service", IsPreview = true)] - [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] - class RestartCommand : Command + protected override Task Run() { - protected override Task Run() + try { - try + var controller = new ServiceController(SeqCliForwarderWindowsService.WindowsServiceName); + + if (controller.Status != ServiceControllerStatus.Stopped) { - var controller = new ServiceController(SeqCliForwarderWindowsService.WindowsServiceName); + Console.WriteLine("Stopping {0}...", controller.ServiceName); + controller.Stop(); if (controller.Status != ServiceControllerStatus.Stopped) { - Console.WriteLine("Stopping {0}...", controller.ServiceName); - controller.Stop(); - - if (controller.Status != ServiceControllerStatus.Stopped) - { - Console.WriteLine("Waiting up to 60 seconds for the service to stop (currently: " + - controller.Status + ")..."); - controller.WaitForStatus(ServiceControllerStatus.Stopped, TimeSpan.FromSeconds(60)); - } - - if (controller.Status != ServiceControllerStatus.Stopped) - { - Console.WriteLine("The service hasn't stopped successfully."); - return Task.FromResult(-1); - } + Console.WriteLine("Waiting up to 60 seconds for the service to stop (currently: " + + controller.Status + ")..."); + controller.WaitForStatus(ServiceControllerStatus.Stopped, TimeSpan.FromSeconds(60)); } - Console.WriteLine("Starting {0}...", controller.ServiceName); - controller.Start(); - - if (controller.Status != ServiceControllerStatus.Running) + if (controller.Status != ServiceControllerStatus.Stopped) { - Console.WriteLine("Waiting up to 15 seconds for the service to start (currently: " + controller.Status + ")..."); - controller.WaitForStatus(ServiceControllerStatus.Running, TimeSpan.FromSeconds(15)); + Console.WriteLine("The service hasn't stopped successfully."); + return Task.FromResult(-1); } + } - if (controller.Status == ServiceControllerStatus.Running) - { - Console.WriteLine("Started."); - return Task.FromResult(0); - } + Console.WriteLine("Starting {0}...", controller.ServiceName); + controller.Start(); - Console.WriteLine("The service hasn't started successfully."); - return Task.FromResult(-1); + if (controller.Status != ServiceControllerStatus.Running) + { + Console.WriteLine("Waiting up to 15 seconds for the service to start (currently: " + controller.Status + ")..."); + controller.WaitForStatus(ServiceControllerStatus.Running, TimeSpan.FromSeconds(15)); } - catch (Exception ex) + + if (controller.Status == ServiceControllerStatus.Running) { - Console.WriteLine(ex.Message); - if (ex.InnerException != null) - Console.WriteLine(ex.InnerException.Message); - return Task.FromResult(1); + Console.WriteLine("Started."); + return Task.FromResult(0); } + + Console.WriteLine("The service hasn't started successfully."); + return Task.FromResult(-1); + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + if (ex.InnerException != null) + Console.WriteLine(ex.InnerException.Message); + return Task.FromResult(1); } } } diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index 4b45e667..7efe2932 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -52,16 +52,16 @@ class RunCommand : Command readonly StoragePathFeature _storagePath; readonly ListenUriFeature _listenUri; readonly ConnectionFeature _connection; - + bool _noLogo; - public RunCommand(SeqConnectionFactory connectionFactory, StoragePathFeature storagePath) + public RunCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory; Options.Add("nologo", _ => _noLogo = true); _listenUri = Enable(); _connection = Enable(); - _storagePath = storagePath; + _storagePath = Enable(); } protected override async Task Run(string[] unrecognized) @@ -81,8 +81,7 @@ protected override async Task Run(string[] unrecognized) SeqCliConfig config; try { - // ISSUE: we can't really rely on the default `SeqCliConfig` path being readable when running as a service. - config = SeqCliConfig.ReadFromFile(_storagePath.ConfigFilePath); + config = RuntimeConfigurationLoader.Load(_storagePath); } catch (Exception ex) { @@ -93,13 +92,13 @@ protected override async Task Run(string[] unrecognized) logger.Fatal(ex, "Failed to load configuration from {ConfigFilePath}", _storagePath.ConfigFilePath); return 1; } - - var connection = _connectionFactory.Connect(_connection); + + var connection = _connectionFactory.Connect(_connection, config); // The API key is passed through separately because `SeqConnection` doesn't expose a batched ingestion // mechanism and so we manually construct `HttpRequestMessage`s deeper in the stack. Nice feature gap to // close at some point! - var (serverUrl, apiKey) = _connectionFactory.GetConnectionDetails(_connection); + var (serverUrl, apiKey) = _connectionFactory.GetConnectionDetails(_connection, config); Log.Logger = CreateLogger( config.Forwarder.Diagnostics.InternalLoggingLevel, diff --git a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs index 6e8f1c7b..0d6d0a0b 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs @@ -20,48 +20,47 @@ using System.Threading.Tasks; using SeqCli.Forwarder.ServiceProcess; -namespace SeqCli.Cli.Commands.Forwarder +namespace SeqCli.Cli.Commands.Forwarder; + +[Command("forwarder", "start", "Start the forwarder Windows service", IsPreview = true)] +[SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] +class StartCommand : Command { - [Command("forwarder", "start", "Start the forwarder Windows service", IsPreview = true)] - [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] - class StartCommand : Command + protected override Task Run() { - protected override Task Run() + try { - try + var controller = new ServiceController(SeqCliForwarderWindowsService.WindowsServiceName); + if (controller.Status != ServiceControllerStatus.Stopped) { - var controller = new ServiceController(SeqCliForwarderWindowsService.WindowsServiceName); - if (controller.Status != ServiceControllerStatus.Stopped) - { - Console.WriteLine("Cannot start {0}, current status is: {1}", controller.ServiceName, controller.Status); - return Task.FromResult(-1); - } - - Console.WriteLine("Starting {0}...", controller.ServiceName); - controller.Start(); + Console.WriteLine("Cannot start {0}, current status is: {1}", controller.ServiceName, controller.Status); + return Task.FromResult(-1); + } - if (controller.Status != ServiceControllerStatus.Running) - { - Console.WriteLine("Waiting up to 15 seconds for the service to start (currently: " + controller.Status + ")..."); - controller.WaitForStatus(ServiceControllerStatus.Running, TimeSpan.FromSeconds(15)); - } + Console.WriteLine("Starting {0}...", controller.ServiceName); + controller.Start(); - if (controller.Status == ServiceControllerStatus.Running) - { - Console.WriteLine("Started."); - return Task.FromResult(0); - } - - Console.WriteLine("The service hasn't started successfully."); - return Task.FromResult(-1); + if (controller.Status != ServiceControllerStatus.Running) + { + Console.WriteLine("Waiting up to 15 seconds for the service to start (currently: " + controller.Status + ")..."); + controller.WaitForStatus(ServiceControllerStatus.Running, TimeSpan.FromSeconds(15)); } - catch (Exception ex) + + if (controller.Status == ServiceControllerStatus.Running) { - Console.WriteLine(ex.Message); - if (ex.InnerException != null) - Console.WriteLine(ex.InnerException.Message); - return Task.FromResult(-1); + Console.WriteLine("Started."); + return Task.FromResult(0); } + + Console.WriteLine("The service hasn't started successfully."); + return Task.FromResult(-1); + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + if (ex.InnerException != null) + Console.WriteLine(ex.InnerException.Message); + return Task.FromResult(-1); } } } diff --git a/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs index 3922e097..f7733460 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs @@ -20,33 +20,32 @@ using System.Threading.Tasks; using SeqCli.Forwarder.ServiceProcess; -namespace SeqCli.Cli.Commands.Forwarder +namespace SeqCli.Cli.Commands.Forwarder; + +[Command("forwarder", "status", "Show the status of the forwarder Windows service", IsPreview = true)] +[SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] +class StatusCommand : Command { - [Command("forwarder", "status", "Show the status of the forwarder Windows service", IsPreview = true)] - [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] - class StatusCommand : Command + protected override Task Run() { - protected override Task Run() + try { - try - { - var controller = new ServiceController(SeqCliForwarderWindowsService.WindowsServiceName); - Console.WriteLine($"The {SeqCliForwarderWindowsService.WindowsServiceName} service is installed and {controller.Status.ToString().ToLowerInvariant()}."); - } - catch (InvalidOperationException) - { - Console.WriteLine($"The {SeqCliForwarderWindowsService.WindowsServiceName} service is not installed."); - } - catch (Exception ex) - { - Console.WriteLine(ex.Message); - if (ex.InnerException != null) - Console.WriteLine(ex.InnerException.Message); - return Task.FromResult(1); - } - + var controller = new ServiceController(SeqCliForwarderWindowsService.WindowsServiceName); + Console.WriteLine($"The {SeqCliForwarderWindowsService.WindowsServiceName} service is installed and {controller.Status.ToString().ToLowerInvariant()}."); + } + catch (InvalidOperationException) + { + Console.WriteLine($"The {SeqCliForwarderWindowsService.WindowsServiceName} service is not installed."); + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + if (ex.InnerException != null) + Console.WriteLine(ex.InnerException.Message); return Task.FromResult(1); } + + return Task.FromResult(1); } } diff --git a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs index 5dc7659c..0dc72a45 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs @@ -20,49 +20,48 @@ using System.Threading.Tasks; using SeqCli.Forwarder.ServiceProcess; -namespace SeqCli.Cli.Commands.Forwarder +namespace SeqCli.Cli.Commands.Forwarder; + +[Command("forwarder", "stop", "Stop the forwarder Windows service", IsPreview = true)] +[SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] +class StopCommand : Command { - [Command("forwarder", "stop", "Stop the forwarder Windows service", IsPreview = true)] - [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] - class StopCommand : Command + protected override Task Run() { - protected override Task Run() + try { - try - { - var controller = new ServiceController(SeqCliForwarderWindowsService.WindowsServiceName); - - if (controller.Status != ServiceControllerStatus.Running) - { - Console.WriteLine("Cannot stop {0}, current status is: {1}", controller.ServiceName, controller.Status); - return Task.FromResult(-1); - } - - Console.WriteLine("Stopping {0}...", controller.ServiceName); - controller.Stop(); + var controller = new ServiceController(SeqCliForwarderWindowsService.WindowsServiceName); - if (controller.Status != ServiceControllerStatus.Stopped) - { - Console.WriteLine("Waiting up to 60 seconds for the service to stop (currently: " + controller.Status + ")..."); - controller.WaitForStatus(ServiceControllerStatus.Stopped, TimeSpan.FromSeconds(60)); - } + if (controller.Status != ServiceControllerStatus.Running) + { + Console.WriteLine("Cannot stop {0}, current status is: {1}", controller.ServiceName, controller.Status); + return Task.FromResult(-1); + } - if (controller.Status == ServiceControllerStatus.Stopped) - { - Console.WriteLine("Stopped."); - return Task.FromResult(0); - } + Console.WriteLine("Stopping {0}...", controller.ServiceName); + controller.Stop(); - Console.WriteLine("The service hasn't stopped successfully."); - return Task.FromResult(-1); + if (controller.Status != ServiceControllerStatus.Stopped) + { + Console.WriteLine("Waiting up to 60 seconds for the service to stop (currently: " + controller.Status + ")..."); + controller.WaitForStatus(ServiceControllerStatus.Stopped, TimeSpan.FromSeconds(60)); } - catch (Exception ex) + + if (controller.Status == ServiceControllerStatus.Stopped) { - Console.WriteLine(ex.Message); - if (ex.InnerException != null) - Console.WriteLine(ex.InnerException.Message); - return Task.FromResult(-1); + Console.WriteLine("Stopped."); + return Task.FromResult(0); } + + Console.WriteLine("The service hasn't stopped successfully."); + return Task.FromResult(-1); + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + if (ex.InnerException != null) + Console.WriteLine(ex.InnerException.Message); + return Task.FromResult(-1); } } } diff --git a/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs index 08a8a678..bf50a2ae 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs @@ -25,9 +25,9 @@ class TruncateCommand : Command readonly StoragePathFeature _storagePath; readonly ConfirmFeature _confirm; - public TruncateCommand(StoragePathFeature storagePath) + public TruncateCommand() { - _storagePath = storagePath; + _storagePath = Enable(); _confirm = Enable(); } diff --git a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs index 8fca384c..eb3299b1 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs @@ -20,30 +20,29 @@ using SeqCli.Forwarder.ServiceProcess; using SeqCli.Forwarder.Util; -namespace SeqCli.Cli.Commands.Forwarder +namespace SeqCli.Cli.Commands.Forwarder; + +[Command("forwarder", "uninstall", "Uninstall the forwarder Windows service", IsPreview = true)] +class UninstallCommand : Command { - [Command("forwarder", "uninstall", "Uninstall the forwarder Windows service", IsPreview = true)] - class UninstallCommand : Command + protected override Task Run() { - protected override Task Run() + try { - try - { - Console.WriteLine("Uninstalling service..."); + Console.WriteLine("Uninstalling service..."); - var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); - var exitCode = CaptiveProcess.Run(sc, $"delete \"{SeqCliForwarderWindowsService.WindowsServiceName}\"", Console.WriteLine, Console.WriteLine); - if (exitCode != 0) - throw new InvalidOperationException($"The `sc.exe delete` call failed with exit code {exitCode}."); + var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); + var exitCode = CaptiveProcess.Run(sc, $"delete \"{SeqCliForwarderWindowsService.WindowsServiceName}\"", Console.WriteLine, Console.WriteLine); + if (exitCode != 0) + throw new InvalidOperationException($"The `sc.exe delete` call failed with exit code {exitCode}."); - Console.WriteLine("Service uninstalled successfully."); - return Task.FromResult(0); - } - catch (Exception ex) - { - Console.WriteLine("Could not uninstall the service: " + ex.Message); - return Task.FromResult(-1); - } + Console.WriteLine("Service uninstalled successfully."); + return Task.FromResult(0); + } + catch (Exception ex) + { + Console.WriteLine("Could not uninstall the service: " + ex.Message); + return Task.FromResult(-1); } } } diff --git a/src/SeqCli/Cli/Commands/HelpCommand.cs b/src/SeqCli/Cli/Commands/HelpCommand.cs index 57b7d64d..3aab532b 100644 --- a/src/SeqCli/Cli/Commands/HelpCommand.cs +++ b/src/SeqCli/Cli/Commands/HelpCommand.cs @@ -37,8 +37,6 @@ public HelpCommand(IEnumerable, CommandMetadata>> availableCo protected override Task Run(string[] unrecognized) { - Console.WriteLine(_pre); - var orderedCommands = _availableCommands .Where(c => !c.Metadata.IsPreview || _pre) .OrderBy(c => c.Metadata.Name) diff --git a/src/SeqCli/Cli/Commands/Index/ListCommand.cs b/src/SeqCli/Cli/Commands/Index/ListCommand.cs index 958451b5..b557f5fd 100644 --- a/src/SeqCli/Cli/Commands/Index/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Index/ListCommand.cs @@ -30,11 +30,12 @@ class ListCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; + readonly StoragePathFeature _storagePath; + string? _id; - public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public ListCommand(SeqConnectionFactory connectionFactory) { - if (config == null) throw new ArgumentNullException(nameof(config)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); Options.Add( @@ -42,19 +43,21 @@ public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) "The id of a single index to list", id => _id = id); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); _connection = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var list = _id is not null ? [await connection.Indexes.FindAsync(_id)] : await connection.Indexes.ListAsync(); - _output.ListEntities(list); + _output.GetOutputFormat(config).ListEntities(list); return 0; } diff --git a/src/SeqCli/Cli/Commands/Index/SuppressCommand.cs b/src/SeqCli/Cli/Commands/Index/SuppressCommand.cs index 5aa495bd..fb00d52e 100644 --- a/src/SeqCli/Cli/Commands/Index/SuppressCommand.cs +++ b/src/SeqCli/Cli/Commands/Index/SuppressCommand.cs @@ -27,11 +27,12 @@ class SuppressCommand : Command { readonly SeqConnectionFactory _connectionFactory; readonly ConnectionFeature _connection; + readonly StoragePathFeature _storagePath; + string? _id; - public SuppressCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public SuppressCommand(SeqConnectionFactory connectionFactory) { - if (config == null) throw new ArgumentNullException(nameof(config)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); Options.Add( @@ -39,6 +40,7 @@ public SuppressCommand(SeqConnectionFactory connectionFactory, SeqCliConfig conf "The id of an index to suppress", id => _id = id); + _storagePath = Enable(); _connection = Enable(); } @@ -50,7 +52,8 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var toSuppress = await connection.Indexes.FindAsync(_id); await connection.Indexes.SuppressAsync(toSuppress); diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index 22f4a181..8109fcfe 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -16,6 +16,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using SeqCli.Ingestion; using SeqCli.Levels; @@ -40,6 +41,7 @@ class IngestCommand : Command readonly SendFailureHandlingFeature _sendFailureHandlingFeature; readonly ConnectionFeature _connection; readonly BatchSizeFeature _batchSize; + readonly StoragePathFeature _storagePath; string? _filter, _level, _message; string _pattern = DefaultPattern; bool _json; @@ -76,6 +78,7 @@ public IngestCommand(SeqConnectionFactory connectionFactory) _sendFailureHandlingFeature = Enable(); _connection = Enable(); _batchSize = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -97,12 +100,13 @@ protected override async Task Run() filter = evt => Seq.Syntax.Expressions.ExpressionResult.IsTrue(eval(evt)); } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); // The API key is passed through separately because `SeqConnection` doesn't expose a batched ingestion // mechanism and so we manually construct `HttpRequestMessage`s deeper in the stack. Nice feature gap to // close at some point! - var (_, apiKey) = _connectionFactory.GetConnectionDetails(_connection); + var (_, apiKey) = _connectionFactory.GetConnectionDetails(_connection, config); var batchSize = _batchSize.Value; foreach (var input in _fileInputFeature.OpenInputs()) diff --git a/src/SeqCli/Cli/Commands/License/ApplyCommand.cs b/src/SeqCli/Cli/Commands/License/ApplyCommand.cs index 7f2970b7..28ac4585 100644 --- a/src/SeqCli/Cli/Commands/License/ApplyCommand.cs +++ b/src/SeqCli/Cli/Commands/License/ApplyCommand.cs @@ -3,6 +3,7 @@ using System.Text; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using SeqCli.Util; using Serilog; @@ -17,7 +18,8 @@ class ApplyCommand : Command { readonly SeqConnectionFactory _connectionFactory; readonly ConnectionFeature _connection; - + readonly StoragePathFeature _storagePath; + string? _certificateFilename; bool _certificateStdin; bool _automaticallyRefresh; @@ -39,6 +41,7 @@ public ApplyCommand(SeqConnectionFactory connectionFactory) "the certificate when the subscription is changed or renewed", _ => _automaticallyRefresh = true); + _storagePath = Enable(); _connection = Enable(); } @@ -71,7 +74,8 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var license = await connection.Licenses.FindCurrentAsync(); license.LicenseText = certificate; license.AutomaticallyRefresh = _automaticallyRefresh; diff --git a/src/SeqCli/Cli/Commands/LogCommand.cs b/src/SeqCli/Cli/Commands/LogCommand.cs index 2d6002ae..a8f0df05 100644 --- a/src/SeqCli/Cli/Commands/LogCommand.cs +++ b/src/SeqCli/Cli/Commands/LogCommand.cs @@ -22,6 +22,7 @@ using Newtonsoft.Json.Linq; using SeqCli.Api; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using Serilog; @@ -35,6 +36,7 @@ class LogCommand : Command readonly SeqConnectionFactory _connectionFactory; readonly PropertiesFeature _properties; readonly ConnectionFeature _connection; + readonly StoragePathFeature _storagePath; string? _message, _level, _timestamp, _exception; public LogCommand(SeqConnectionFactory connectionFactory) @@ -63,6 +65,7 @@ public LogCommand(SeqConnectionFactory connectionFactory) _properties = Enable(); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -103,8 +106,9 @@ protected override async Task Run() content = new StringContent(builder.ToString(), Encoding.UTF8, ApiConstants.ClefMediaType); } - var connection = _connectionFactory.Connect(_connection); - var (_, apiKey) = _connectionFactory.GetConnectionDetails(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); + var (_, apiKey) = _connectionFactory.GetConnectionDetails(_connection, config); var request = new HttpRequestMessage(HttpMethod.Post, ApiConstants.IngestionEndpoint) {Content = content}; if (apiKey != null) diff --git a/src/SeqCli/Cli/Commands/Node/HealthCommand.cs b/src/SeqCli/Cli/Commands/Node/HealthCommand.cs index f7710fdf..0e1ce0a5 100644 --- a/src/SeqCli/Cli/Commands/Node/HealthCommand.cs +++ b/src/SeqCli/Cli/Commands/Node/HealthCommand.cs @@ -37,32 +37,35 @@ class HealthCommand : Command readonly WaitUntilHealthyFeature _waitUntilHealthy; readonly TimeoutFeature _timeout; readonly OutputFormatFeature _output; - - public HealthCommand(SeqConnectionFactory connectionFactory, SeqCliOutputConfig seqCliOutputConfig) + readonly StoragePathFeature _storagePath; + + public HealthCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory; _waitUntilHealthy = Enable(new WaitUntilHealthyFeature("node")); _timeout = Enable(new TimeoutFeature()); _connection = Enable(); - _output = Enable(new OutputFormatFeature(seqCliOutputConfig)); + _output = Enable(); + _storagePath = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var timeout = _timeout.ApplyTimeout(connection.Client.HttpClient); if (_waitUntilHealthy.ShouldWait) { - return await RunUntilHealthy(connection, timeout ?? TimeSpan.FromSeconds(30)); + return await RunUntilHealthy(connection, timeout ?? TimeSpan.FromSeconds(30), _output.GetOutputFormat(config)); } - return await RunOnce(connection); + return await RunOnce(connection, _output.GetOutputFormat(config)); } - async Task RunUntilHealthy(SeqConnection connection, TimeSpan timeout) + async Task RunUntilHealthy(SeqConnection connection, TimeSpan timeout, OutputFormat outputFormat) { using var ct = new CancellationTokenSource(timeout); @@ -76,7 +79,7 @@ async Task RunUntilHealthy(SeqConnection connection, TimeSpan timeout) { while (true) { - if (await RunOnce(connection) == 0) + if (await RunOnce(connection, outputFormat) == 0) { return 0; } @@ -91,7 +94,7 @@ async Task RunUntilHealthy(SeqConnection connection, TimeSpan timeout) } } - async Task RunOnce(SeqConnection connection) + async Task RunOnce(SeqConnection connection, OutputFormat outputFormat) { try { @@ -104,17 +107,17 @@ async Task RunOnce(SeqConnection connection) Log.Information("{HeaderName}: {HeaderValue}", key, value); } - if (_output.Json) + if (outputFormat.Json) { var shouldBeJson = await response.Content.ReadAsStringAsync(); try { var obj = JsonConvert.DeserializeObject(shouldBeJson) ?? throw new InvalidDataException(); - _output.WriteObject(obj); + outputFormat.WriteObject(obj); } catch { - _output.WriteObject(new { Response = shouldBeJson }); + outputFormat.WriteObject(new { Response = shouldBeJson }); } } else diff --git a/src/SeqCli/Cli/Commands/Node/ListCommand.cs b/src/SeqCli/Cli/Commands/Node/ListCommand.cs index 1dc0002f..10f9cc8d 100644 --- a/src/SeqCli/Cli/Commands/Node/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Node/ListCommand.cs @@ -29,10 +29,11 @@ class ListCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; - + readonly StoragePathFeature _storagePath; + string? _name, _id; - public ListCommand(SeqConnectionFactory connectionFactory, SeqCliOutputConfig seqCliOutputConfig) + public ListCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -46,20 +47,22 @@ public ListCommand(SeqConnectionFactory connectionFactory, SeqCliOutputConfig se "The id of a single cluster node to list", id => _id = id); - _output = Enable(new OutputFormatFeature(seqCliOutputConfig)); + _output = Enable(); + _storagePath = Enable(); _connection = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); - var list = _id != null ? - new[] { await connection.Cluster.FindAsync(_id) } : + var list = _id != null ? [await connection.Cluster.FindAsync(_id)] + : (await connection.Cluster.ListAsync()) .Where(n => _name == null || _name == n.Name); - _output.ListEntities(list); + _output.GetOutputFormat(config).ListEntities(list); return 0; } diff --git a/src/SeqCli/Cli/Commands/PrintCommand.cs b/src/SeqCli/Cli/Commands/PrintCommand.cs index 84013b5a..883685e0 100644 --- a/src/SeqCli/Cli/Commands/PrintCommand.cs +++ b/src/SeqCli/Cli/Commands/PrintCommand.cs @@ -35,7 +35,7 @@ class PrintCommand : Command readonly FileInputFeature _fileInputFeature; readonly InvalidDataHandlingFeature _invalidDataHandlingFeature; - string? _filter, _template = OutputFormatFeature.DefaultOutputTemplate; + string? _filter, _template = OutputFormat.DefaultOutputTemplate; bool _noColor, _forceColor; public PrintCommand(SeqCliOutputConfig seqCliOutputConfig) @@ -70,14 +70,14 @@ var applyThemeToRedirectedOutput var theme = _noColor ? ConsoleTheme.None - : applyThemeToRedirectedOutput ? OutputFormatFeature.DefaultAnsiTheme - : OutputFormatFeature.DefaultTheme; + : applyThemeToRedirectedOutput ? OutputFormat.DefaultAnsiTheme + : OutputFormat.DefaultTheme; var outputConfiguration = new LoggerConfiguration() .MinimumLevel.Is(LevelAlias.Minimum) .Enrich.With() .WriteTo.Console( - outputTemplate: _template ?? OutputFormatFeature.DefaultOutputTemplate, + outputTemplate: _template ?? OutputFormat.DefaultOutputTemplate, theme: theme, applyThemeToRedirectedOutput: applyThemeToRedirectedOutput); diff --git a/src/SeqCli/Cli/Commands/Profile/CreateCommand.cs b/src/SeqCli/Cli/Commands/Profile/CreateCommand.cs index e5be2268..ba2dfc5d 100644 --- a/src/SeqCli/Cli/Commands/Profile/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Profile/CreateCommand.cs @@ -13,8 +13,8 @@ class CreateCommand : Command { string? _url, _apiKey, _name; readonly StoragePathFeature _storagePath; - - public CreateCommand(StoragePathFeature storagePath) + + public CreateCommand() { Options.Add("n=|name=", "The name of the connection profile", @@ -28,7 +28,7 @@ public CreateCommand(StoragePathFeature storagePath) "The API key to use when connecting to the server, if required", v => _apiKey = ArgumentString.Normalize(v)); - _storagePath = storagePath; + _storagePath = Enable(); } protected override Task Run() diff --git a/src/SeqCli/Cli/Commands/Profile/ListCommand.cs b/src/SeqCli/Cli/Commands/Profile/ListCommand.cs index fcdff3df..c43d3324 100644 --- a/src/SeqCli/Cli/Commands/Profile/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Profile/ListCommand.cs @@ -11,10 +11,10 @@ namespace SeqCli.Cli.Commands.Profile; class ListCommand : Command { readonly StoragePathFeature _storagePath; - - public ListCommand(StoragePathFeature storagePath) + + public ListCommand() { - _storagePath = storagePath; + _storagePath = Enable(); } protected override Task Run() diff --git a/src/SeqCli/Cli/Commands/Profile/RemoveCommand.cs b/src/SeqCli/Cli/Commands/Profile/RemoveCommand.cs index bcd69ea4..244b520f 100644 --- a/src/SeqCli/Cli/Commands/Profile/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/Profile/RemoveCommand.cs @@ -14,13 +14,13 @@ class RemoveCommand : Command string? _name; readonly StoragePathFeature _storagePath; - public RemoveCommand(StoragePathFeature storagePathPath) + public RemoveCommand() { Options.Add("n=|name=", "The name of the connection profile to remove", v => _name = ArgumentString.Normalize(v)); - _storagePath = storagePathPath; + _storagePath = Enable(); } protected override Task Run() diff --git a/src/SeqCli/Cli/Commands/QueryCommand.cs b/src/SeqCli/Cli/Commands/QueryCommand.cs index 2dfd8058..3cafab0f 100644 --- a/src/SeqCli/Cli/Commands/QueryCommand.cs +++ b/src/SeqCli/Cli/Commands/QueryCommand.cs @@ -34,18 +34,19 @@ class QueryCommand : Command readonly DateRangeFeature _range; readonly SignalExpressionFeature _signal; readonly TimeoutFeature _timeout; + readonly StoragePathFeature _storagePath; string? _query; bool _trace; - public QueryCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public QueryCommand(SeqConnectionFactory connectionFactory) { - if (config == null) throw new ArgumentNullException(nameof(config)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); Options.Add("q=|query=", "The query to execute", v => _query = v); _range = Enable(); _signal = Enable(); _timeout = Enable(); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); Options.Add("trace", "Enable detailed (server-side) query tracing", _ => _trace = true); _connection = Enable(); } @@ -58,11 +59,13 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var timeout = _timeout.ApplyTimeout(connection.Client.HttpClient); - - if (_output.Json) + + var output = _output.GetOutputFormat(config); + if (output.Json) { var result = await connection.Data.QueryAsync(_query, _range.Start, _range.End, _signal.Signal, timeout: timeout, trace: _trace); @@ -72,7 +75,7 @@ protected override async Task Run() else { var result = await connection.Data.QueryCsvAsync(_query, _range.Start, _range.End, _signal.Signal, timeout: timeout, trace: _trace); - _output.WriteCsv(result); + output.WriteCsv(result); } return 0; diff --git a/src/SeqCli/Cli/Commands/RetentionPolicy/CreateCommand.cs b/src/SeqCli/Cli/Commands/RetentionPolicy/CreateCommand.cs index 6ead5878..5cd947c0 100644 --- a/src/SeqCli/Cli/Commands/RetentionPolicy/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/RetentionPolicy/CreateCommand.cs @@ -33,12 +33,13 @@ class CreateCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; - + readonly StoragePathFeature _storagePath; + string? _afterDuration; bool _deleteAllEvents; string? _deleteMatchingSignal; - public CreateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public CreateCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -62,12 +63,14 @@ public CreateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config ); _connection = Enable(); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); SignalExpressionPart? removedSignalExpression; @@ -106,7 +109,7 @@ protected override async Task Run() policy = await connection.RetentionPolicies.AddAsync(policy); - _output.WriteEntity(policy); + _output.GetOutputFormat(config).WriteEntity(policy); return 0; } diff --git a/src/SeqCli/Cli/Commands/RetentionPolicy/ListCommand.cs b/src/SeqCli/Cli/Commands/RetentionPolicy/ListCommand.cs index 710ae634..b9a45e1c 100644 --- a/src/SeqCli/Cli/Commands/RetentionPolicy/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/RetentionPolicy/ListCommand.cs @@ -13,7 +13,6 @@ // limitations under the License. using System; -using System.Linq; using System.Threading.Tasks; using SeqCli.Cli.Features; using SeqCli.Config; @@ -28,12 +27,12 @@ class ListCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; - + readonly StoragePathFeature _storagePath; + string? _id; - public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public ListCommand(SeqConnectionFactory connectionFactory) { - if (config == null) throw new ArgumentNullException(nameof(config)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); Options.Add( @@ -41,20 +40,22 @@ public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) "The id of a single retention policy to list", id => _id = id); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); _connection = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var list = _id != null ? [await connection.RetentionPolicies.FindAsync(_id)] : (await connection.RetentionPolicies.ListAsync()) .ToArray(); - - _output.ListEntities(list); + + _output.GetOutputFormat(config).ListEntities(list); return 0; } diff --git a/src/SeqCli/Cli/Commands/RetentionPolicy/RemoveCommand.cs b/src/SeqCli/Cli/Commands/RetentionPolicy/RemoveCommand.cs index e083d779..2fa961f9 100644 --- a/src/SeqCli/Cli/Commands/RetentionPolicy/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/RetentionPolicy/RemoveCommand.cs @@ -15,6 +15,7 @@ using System; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using Serilog; @@ -27,7 +28,8 @@ class RemoveCommand : Command readonly SeqConnectionFactory _connectionFactory; readonly ConnectionFeature _connection; - + readonly StoragePathFeature _storagePath; + string? _id; public RemoveCommand(SeqConnectionFactory connectionFactory) @@ -40,6 +42,7 @@ public RemoveCommand(SeqConnectionFactory connectionFactory) id => _id = id); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -50,7 +53,8 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var toRemove = await connection.RetentionPolicies.FindAsync(_id); diff --git a/src/SeqCli/Cli/Commands/Sample/IngestCommand.cs b/src/SeqCli/Cli/Commands/Sample/IngestCommand.cs index 5058e40f..b4cbd2b7 100644 --- a/src/SeqCli/Cli/Commands/Sample/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/Sample/IngestCommand.cs @@ -16,6 +16,7 @@ using System.Linq; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using SeqCli.Sample.Loader; @@ -30,7 +31,8 @@ class IngestCommand : Command readonly ConnectionFeature _connection; readonly ConfirmFeature _confirm; readonly BatchSizeFeature _batchSize; - + readonly StoragePathFeature _storagePath; + bool _quiet; bool _setup; int _simulations = 1; @@ -47,11 +49,13 @@ public IngestCommand(SeqConnectionFactory connectionFactory) v => _simulations = int.Parse(v)); _batchSize = Enable(); + _storagePath = Enable(); } protected override async Task Run() { - var (url, apiKey) = _connectionFactory.GetConnectionDetails(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var (url, apiKey) = _connectionFactory.GetConnectionDetails(_connection, config); var batchSize = _batchSize.Value; if (!_confirm.TryConfirm(_setup @@ -62,7 +66,7 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection); + var connection = _connectionFactory.Connect(_connection, config); if (_setup) { diff --git a/src/SeqCli/Cli/Commands/Sample/SetupCommand.cs b/src/SeqCli/Cli/Commands/Sample/SetupCommand.cs index 6c55dfb6..0f5dadea 100644 --- a/src/SeqCli/Cli/Commands/Sample/SetupCommand.cs +++ b/src/SeqCli/Cli/Commands/Sample/SetupCommand.cs @@ -23,6 +23,7 @@ using SeqCli.Templates.Import; using SeqCli.Util; using Seq.Api; +using SeqCli.Config; // ReSharper disable once UnusedType.Global @@ -36,7 +37,8 @@ class SetupCommand : Command readonly ConnectionFeature _connection; readonly ConfirmFeature _confirm; - + readonly StoragePathFeature _storagePath; + public SetupCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -46,13 +48,15 @@ public SetupCommand(SeqConnectionFactory connectionFactory) _confirm = Enable(); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); - var (url, _) = _connectionFactory.GetConnectionDetails(_connection); + var (url, _) = _connectionFactory.GetConnectionDetails(_connection, config); if (!_confirm.TryConfirm($"This will apply sample configuration items to the Seq server at {url}.")) { await Console.Error.WriteLineAsync("Canceled by user."); @@ -64,8 +68,10 @@ protected override async Task Run() internal static async Task ImportTemplates(SeqConnection connection) { - var templateArgs = new Dictionary(); - templateArgs["ownerId"] = new JsonTemplateNull(); + var templateArgs = new Dictionary + { + ["ownerId"] = new JsonTemplateNull() + }; var templatesPath = Content.GetPath(Path.Combine("Sample", "Templates")); var templateFiles = Directory.GetFiles(templatesPath); diff --git a/src/SeqCli/Cli/Commands/SearchCommand.cs b/src/SeqCli/Cli/Commands/SearchCommand.cs index 001bca4d..016d36c2 100644 --- a/src/SeqCli/Cli/Commands/SearchCommand.cs +++ b/src/SeqCli/Cli/Commands/SearchCommand.cs @@ -39,12 +39,13 @@ class SearchCommand : Command readonly OutputFormatFeature _output; readonly DateRangeFeature _range; readonly SignalExpressionFeature _signal; + readonly StoragePathFeature _storagePath; string? _filter; int _count = 1; int _httpClientTimeout = 100000; bool _trace, _noWebSockets; - public SearchCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public SearchCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -58,7 +59,8 @@ public SearchCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config v => _count = int.Parse(v, CultureInfo.InvariantCulture)); _range = Enable(); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); _signal = Enable(); Options.Add( @@ -77,8 +79,9 @@ protected override async Task Run() { try { - await using var output = _output.CreateOutputLogger(); - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + await using var output = _output.GetOutputFormat(config).CreateOutputLogger(); + var connection = _connectionFactory.Connect(_connection, config); connection.Client.HttpClient.Timeout = TimeSpan.FromMilliseconds(_httpClientTimeout); string? filter = null; diff --git a/src/SeqCli/Cli/Commands/Settings/ClearCommand.cs b/src/SeqCli/Cli/Commands/Settings/ClearCommand.cs index 2a3ac781..08d79ba1 100644 --- a/src/SeqCli/Cli/Commands/Settings/ClearCommand.cs +++ b/src/SeqCli/Cli/Commands/Settings/ClearCommand.cs @@ -15,6 +15,7 @@ using System; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; namespace SeqCli.Cli.Commands.Settings; @@ -26,18 +27,21 @@ class ClearCommand: Command readonly ConnectionFeature _connection; readonly SettingNameFeature _name; - + readonly StoragePathFeature _storagePath; + public ClearCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); _name = Enable(); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var setting = await connection.Settings.FindNamedAsync(_name.Name); setting.Value = null; diff --git a/src/SeqCli/Cli/Commands/Settings/SetCommand.cs b/src/SeqCli/Cli/Commands/Settings/SetCommand.cs index 8270f87e..4c3f3051 100644 --- a/src/SeqCli/Cli/Commands/Settings/SetCommand.cs +++ b/src/SeqCli/Cli/Commands/Settings/SetCommand.cs @@ -15,6 +15,7 @@ using System; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using Serilog; @@ -27,6 +28,7 @@ class SetCommand: Command readonly ConnectionFeature _connection; readonly SettingNameFeature _name; + readonly StoragePathFeature _storagePath; string? _value; bool _valueSpecified, _readValueFromStdin; @@ -51,6 +53,7 @@ public SetCommand(SeqConnectionFactory connectionFactory) _ => _readValueFromStdin = true); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -60,8 +63,9 @@ protected override async Task Run() Log.Error("A value must be supplied with either `--value=VALUE` or `--value-stdin`."); return 1; } - - var connection = _connectionFactory.Connect(_connection); + + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var setting = await connection.Settings.FindNamedAsync(_name.Name); setting.Value = ReadValue(); diff --git a/src/SeqCli/Cli/Commands/Settings/ShowCommand.cs b/src/SeqCli/Cli/Commands/Settings/ShowCommand.cs index 38582e80..93d1bd9e 100644 --- a/src/SeqCli/Cli/Commands/Settings/ShowCommand.cs +++ b/src/SeqCli/Cli/Commands/Settings/ShowCommand.cs @@ -16,6 +16,7 @@ using System.Globalization; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; namespace SeqCli.Cli.Commands.Settings; @@ -27,18 +28,21 @@ class ShowCommand: Command readonly ConnectionFeature _connection; readonly SettingNameFeature _name; - + readonly StoragePathFeature _storagePath; + public ShowCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); _name = Enable(); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var setting = await connection.Settings.FindNamedAsync(_name.Name); diff --git a/src/SeqCli/Cli/Commands/Shared/UpdateCommand.cs b/src/SeqCli/Cli/Commands/Shared/UpdateCommand.cs index df555ee8..fdf5bc44 100644 --- a/src/SeqCli/Cli/Commands/Shared/UpdateCommand.cs +++ b/src/SeqCli/Cli/Commands/Shared/UpdateCommand.cs @@ -16,6 +16,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using SeqCli.Templates.Ast; using SeqCli.Templates.Import; @@ -30,6 +31,7 @@ abstract class UpdateCommand: Command readonly SeqConnectionFactory _connectionFactory; readonly ConnectionFeature _connection; + readonly StoragePathFeature _storagePath; readonly string _resourceGroupName; readonly string _entityName; @@ -53,11 +55,13 @@ protected UpdateCommand(SeqConnectionFactory connectionFactory, string commandGr _ => _jsonStdin = true); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); if (_json == null && !_jsonStdin) { diff --git a/src/SeqCli/Cli/Commands/Signal/CreateCommand.cs b/src/SeqCli/Cli/Commands/Signal/CreateCommand.cs index eb6ae8ff..539edfb2 100644 --- a/src/SeqCli/Cli/Commands/Signal/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Signal/CreateCommand.cs @@ -33,13 +33,14 @@ class CreateCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; - + readonly StoragePathFeature _storagePath; + readonly List _columns = new(); string? _title, _description, _filter, _group; bool _isProtected, _noGrouping; - public CreateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public CreateCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -79,12 +80,14 @@ public CreateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config _ => _isProtected = true); _connection = Enable(); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var signal = await connection.Signals.TemplateAsync(); signal.OwnerId = null; @@ -124,7 +127,7 @@ protected override async Task Run() signal = await connection.Signals.AddAsync(signal); - _output.WriteEntity(signal); + _output.GetOutputFormat(config).WriteEntity(signal); return 0; } diff --git a/src/SeqCli/Cli/Commands/Signal/ImportCommand.cs b/src/SeqCli/Cli/Commands/Signal/ImportCommand.cs index eb0750dc..66daf445 100644 --- a/src/SeqCli/Cli/Commands/Signal/ImportCommand.cs +++ b/src/SeqCli/Cli/Commands/Signal/ImportCommand.cs @@ -32,7 +32,8 @@ class ImportCommand : Command readonly FileInputFeature _fileInputFeature; readonly EntityOwnerFeature _entityOwner; readonly ConnectionFeature _connection; - + readonly StoragePathFeature _storagePath; + bool _merge; readonly JsonSerializer _serializer = JsonSerializer.Create( @@ -40,9 +41,8 @@ class ImportCommand : Command Converters = { new StringEnumConverter() } }); - public ImportCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public ImportCommand(SeqConnectionFactory connectionFactory) { - if (config == null) throw new ArgumentNullException(nameof(config)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); Options.Add( @@ -53,11 +53,13 @@ public ImportCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config _fileInputFeature = Enable(new FileInputFeature("File to import")); _entityOwner = Enable(new EntityOwnerFeature("signal", "import", "imported")); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); using var input = _fileInputFeature.OpenSingleInput(); var line = await input.ReadLineAsync(); diff --git a/src/SeqCli/Cli/Commands/Signal/ListCommand.cs b/src/SeqCli/Cli/Commands/Signal/ListCommand.cs index a4ef83df..bfbdaa11 100644 --- a/src/SeqCli/Cli/Commands/Signal/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Signal/ListCommand.cs @@ -30,28 +30,30 @@ class ListCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly EntityOwnerFeature _entityOwner; - - public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + readonly StoragePathFeature _storagePath; + + public ListCommand(SeqConnectionFactory connectionFactory) { - if (config == null) throw new ArgumentNullException(nameof(config)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); _entityIdentity = Enable(new EntityIdentityFeature("signal", "list")); _entityOwner = Enable(new EntityOwnerFeature("signal", "list", "listed", _entityIdentity)); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); _connection = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); - var list = _entityIdentity.Id != null ? - new[] { await connection.Signals.FindAsync(_entityIdentity.Id) } : + var list = _entityIdentity.Id != null ? [await connection.Signals.FindAsync(_entityIdentity.Id)] + : (await connection.Signals.ListAsync(ownerId: _entityOwner.OwnerId, shared: _entityOwner.IncludeShared)) .Where(signal => _entityIdentity.Title == null || _entityIdentity.Title == signal.Title); - _output.ListEntities(list); + _output.GetOutputFormat(config).ListEntities(list); return 0; } diff --git a/src/SeqCli/Cli/Commands/Signal/RemoveCommand.cs b/src/SeqCli/Cli/Commands/Signal/RemoveCommand.cs index 5a5d7e45..4727e91f 100644 --- a/src/SeqCli/Cli/Commands/Signal/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/Signal/RemoveCommand.cs @@ -16,6 +16,7 @@ using System.Linq; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using Serilog; @@ -30,7 +31,8 @@ class RemoveCommand : Command readonly EntityIdentityFeature _entityIdentity; readonly ConnectionFeature _connection; readonly EntityOwnerFeature _entityOwner; - + readonly StoragePathFeature _storagePath; + public RemoveCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -38,6 +40,7 @@ public RemoveCommand(SeqConnectionFactory connectionFactory) _entityIdentity = Enable(new EntityIdentityFeature("signal", "remove")); _entityOwner = Enable(new EntityOwnerFeature("signal", "remove", "removed", _entityIdentity)); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -48,7 +51,8 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var toRemove = _entityIdentity.Id != null ? [await connection.Signals.FindAsync(_entityIdentity.Id)] : diff --git a/src/SeqCli/Cli/Commands/TailCommand.cs b/src/SeqCli/Cli/Commands/TailCommand.cs index 0eab6644..1b7553c6 100644 --- a/src/SeqCli/Cli/Commands/TailCommand.cs +++ b/src/SeqCli/Cli/Commands/TailCommand.cs @@ -30,9 +30,10 @@ class TailCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly SignalExpressionFeature _signal; + readonly StoragePathFeature _storagePath; string? _filter; - public TailCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public TailCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -41,7 +42,8 @@ public TailCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) "An optional server-side filter to apply to the stream, for example `@Level = 'Error'`", v => _filter = v); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); _signal = Enable(); _connection = Enable(); } @@ -50,8 +52,9 @@ protected override async Task Run() { var cancel = new CancellationTokenSource(); Console.CancelKeyPress += (_,_) => cancel.Cancel(); - - var connection = _connectionFactory.Connect(_connection); + + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); string? strict = null; if (!string.IsNullOrWhiteSpace(_filter)) @@ -60,7 +63,7 @@ protected override async Task Run() strict = converted.StrictExpression; } - await using var output = _output.CreateOutputLogger(); + await using var output = _output.GetOutputFormat(config).CreateOutputLogger(); try { diff --git a/src/SeqCli/Cli/Commands/Template/ExportCommand.cs b/src/SeqCli/Cli/Commands/Template/ExportCommand.cs index d9b654ec..e4bda0a0 100644 --- a/src/SeqCli/Cli/Commands/Template/ExportCommand.cs +++ b/src/SeqCli/Cli/Commands/Template/ExportCommand.cs @@ -3,6 +3,7 @@ using System.IO; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using SeqCli.Templates.Export; using SeqCli.Util; @@ -18,7 +19,8 @@ class ExportCommand : Command { readonly SeqConnectionFactory _connectionFactory; readonly ConnectionFeature _connection; - + readonly StoragePathFeature _storagePath; + readonly HashSet _include = new(); string? _outputDir = "."; @@ -39,6 +41,7 @@ public ExportCommand(SeqConnectionFactory connectionFactory) i => _include.Add(ArgumentString.Normalize(i))); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -55,7 +58,8 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var export = new TemplateSetExporter(connection, _include, _outputDir); await export.ExportTemplateSet(); diff --git a/src/SeqCli/Cli/Commands/Template/ImportCommand.cs b/src/SeqCli/Cli/Commands/Template/ImportCommand.cs index eb611e02..88b2d607 100644 --- a/src/SeqCli/Cli/Commands/Template/ImportCommand.cs +++ b/src/SeqCli/Cli/Commands/Template/ImportCommand.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using SeqCli.Templates.Ast; using SeqCli.Templates.Export; @@ -25,7 +26,8 @@ class ImportCommand : Command readonly SeqConnectionFactory _connectionFactory; readonly ConnectionFeature _connection; readonly PropertiesFeature _args; - + readonly StoragePathFeature _storagePath; + string? _inputDir = "."; string? _stateFile; bool _merge; @@ -54,6 +56,7 @@ public ImportCommand(SeqConnectionFactory connectionFactory) _args = Enable(new PropertiesFeature("g", "arg", "Template arguments, e.g. `-g ownerId=user-314159`")); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -97,7 +100,8 @@ protected override async Task Run() _ => throw new NotSupportedException("Unexpected property type.") })); - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var err = await TemplateSetImporter.ImportAsync(templates, connection, args, state, _merge); await TemplateImportState.SaveAsync(stateFile, state); diff --git a/src/SeqCli/Cli/Commands/User/CreateCommand.cs b/src/SeqCli/Cli/Commands/User/CreateCommand.cs index 0b871d24..21044540 100644 --- a/src/SeqCli/Cli/Commands/User/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/User/CreateCommand.cs @@ -32,11 +32,12 @@ class CreateCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; - + readonly StoragePathFeature _storagePath; + string? _username, _displayName, _roleTitle, _filter, _emailAddress, _password; bool _passwordStdin, _noPasswordChange; - public CreateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public CreateCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -81,12 +82,14 @@ public CreateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config _ => _noPasswordChange = true); _connection = Enable(); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var user = await connection.Users.TemplateAsync(); @@ -148,7 +151,7 @@ protected override async Task Run() user = await connection.Users.AddAsync(user); - _output.WriteEntity(user); + _output.GetOutputFormat(config).WriteEntity(user); return 0; } diff --git a/src/SeqCli/Cli/Commands/User/ListCommand.cs b/src/SeqCli/Cli/Commands/User/ListCommand.cs index d3220d1c..43417073 100644 --- a/src/SeqCli/Cli/Commands/User/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/User/ListCommand.cs @@ -29,27 +29,29 @@ class ListCommand : Command readonly UserIdentityFeature _userIdentity; readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; - - public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + readonly StoragePathFeature _storagePath; + + public ListCommand(SeqConnectionFactory connectionFactory) { - if (config == null) throw new ArgumentNullException(nameof(config)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); _userIdentity = Enable(new UserIdentityFeature("list")); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); _connection = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); - var list = _userIdentity.Id != null ? - new[] { await connection.Users.FindAsync(_userIdentity.Id) } : + var list = _userIdentity.Id != null ? [await connection.Users.FindAsync(_userIdentity.Id)] + : (await connection.Users.ListAsync()) .Where(u => _userIdentity.Name == null || _userIdentity.Name == u.Username); - _output.ListEntities(list); + _output.GetOutputFormat(config).ListEntities(list); return 0; } diff --git a/src/SeqCli/Cli/Commands/User/RemoveCommand.cs b/src/SeqCli/Cli/Commands/User/RemoveCommand.cs index 782f90d9..ece8898a 100644 --- a/src/SeqCli/Cli/Commands/User/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/User/RemoveCommand.cs @@ -16,6 +16,7 @@ using System.Linq; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using Serilog; @@ -29,13 +30,15 @@ class RemoveCommand : Command readonly UserIdentityFeature _userIdentity; readonly ConnectionFeature _connection; - + readonly StoragePathFeature _storagePath; + public RemoveCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); _userIdentity = Enable(new UserIdentityFeature("remove")); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -46,7 +49,8 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var toRemove = _userIdentity.Id != null ? [await connection.Users.FindAsync(_userIdentity.Id)] : diff --git a/src/SeqCli/Cli/Commands/Workspace/CreateCommand.cs b/src/SeqCli/Cli/Commands/Workspace/CreateCommand.cs index f0c3262e..21c8457d 100644 --- a/src/SeqCli/Cli/Commands/Workspace/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Workspace/CreateCommand.cs @@ -19,12 +19,13 @@ class CreateCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; - + readonly StoragePathFeature _storagePath; + string? _title, _description; bool _isProtected; readonly List _include = new(); - public CreateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + public CreateCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -49,12 +50,14 @@ public CreateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config _ => _isProtected = true); _connection = Enable(); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var workspace = await connection.Workspaces.TemplateAsync(); workspace.OwnerId = null; @@ -69,7 +72,7 @@ protected override async Task Run() workspace = await connection.Workspaces.AddAsync(workspace); - _output.WriteEntity(workspace); + _output.GetOutputFormat(config).WriteEntity(workspace); return 0; } diff --git a/src/SeqCli/Cli/Commands/Workspace/ListCommand.cs b/src/SeqCli/Cli/Commands/Workspace/ListCommand.cs index 7b3ed432..84c2f777 100644 --- a/src/SeqCli/Cli/Commands/Workspace/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Workspace/ListCommand.cs @@ -16,28 +16,30 @@ class ListCommand : Command readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly EntityOwnerFeature _entityOwner; - - public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) + readonly StoragePathFeature _storagePath; + + public ListCommand(SeqConnectionFactory connectionFactory) { - if (config == null) throw new ArgumentNullException(nameof(config)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); _entityIdentity = Enable(new EntityIdentityFeature("workspace", "list")); _entityOwner = Enable(new EntityOwnerFeature("workspace", "list", "listed", _entityIdentity)); - _output = Enable(new OutputFormatFeature(config.Output)); + _output = Enable(); + _storagePath = Enable(); _connection = Enable(); } protected override async Task Run() { - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); - var list = _entityIdentity.Id != null ? - new[] { await connection.Workspaces.FindAsync(_entityIdentity.Id) } : + var list = _entityIdentity.Id != null ? [await connection.Workspaces.FindAsync(_entityIdentity.Id)] + : (await connection.Workspaces.ListAsync(ownerId: _entityOwner.OwnerId, shared: _entityOwner.IncludeShared)) .Where(workspace => _entityIdentity.Title == null || _entityIdentity.Title == workspace.Title); - _output.ListEntities(list); + _output.GetOutputFormat(config).ListEntities(list); return 0; } diff --git a/src/SeqCli/Cli/Commands/Workspace/RemoveCommand.cs b/src/SeqCli/Cli/Commands/Workspace/RemoveCommand.cs index b61b25aa..68d5b751 100644 --- a/src/SeqCli/Cli/Commands/Workspace/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/Workspace/RemoveCommand.cs @@ -2,6 +2,7 @@ using System.Linq; using System.Threading.Tasks; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using Serilog; @@ -16,7 +17,8 @@ class RemoveCommand : Command readonly EntityIdentityFeature _entityIdentity; readonly ConnectionFeature _connection; readonly EntityOwnerFeature _entityOwner; - + readonly StoragePathFeature _storagePath; + public RemoveCommand(SeqConnectionFactory connectionFactory) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); @@ -24,6 +26,7 @@ public RemoveCommand(SeqConnectionFactory connectionFactory) _entityIdentity = Enable(new EntityIdentityFeature("workspace", "remove")); _entityOwner = Enable(new EntityOwnerFeature("workspace", "remove", "removed", _entityIdentity)); _connection = Enable(); + _storagePath = Enable(); } protected override async Task Run() @@ -34,7 +37,8 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = _connectionFactory.Connect(_connection, config); var toRemove = _entityIdentity.Id != null ? [await connection.Workspaces.FindAsync(_entityIdentity.Id)] : diff --git a/src/SeqCli/Cli/Features/OutputFormatFeature.cs b/src/SeqCli/Cli/Features/OutputFormatFeature.cs index 2ea555d9..a1c7fa5a 100644 --- a/src/SeqCli/Cli/Features/OutputFormatFeature.cs +++ b/src/SeqCli/Cli/Features/OutputFormatFeature.cs @@ -32,20 +32,40 @@ namespace SeqCli.Cli.Features; class OutputFormatFeature : CommandFeature { - public const string DefaultOutputTemplate = + bool _json; + bool? _noColor, _forceColor; + + public OutputFormat GetOutputFormat(SeqCliConfig config) + { + return new OutputFormat(_json, _noColor ?? config.Output.DisableColor, _forceColor ?? config.Output.ForceColor); + } + + public override void Enable(OptionSet options) + { + options.Add( + "json", + "Print output in newline-delimited JSON (the default is plain text)", + _ => _json = true); + + options.Add("no-color", "Don't colorize text output", _ => _noColor = true); + + options.Add("force-color", + "Force redirected output to have ANSI color (unless `--no-color` is also specified)", + _ => _forceColor = true); + } +} + +sealed class OutputFormat(bool _json, bool _noColor, bool _forceColor) +{ + public const string DefaultOutputTemplate = "[{Timestamp:o} {Level:u3}] {Message:lj} {Properties:j}{NewLine}{Exception}"; public static readonly ConsoleTheme DefaultAnsiTheme = AnsiConsoleTheme.Code; - public static readonly ConsoleTheme DefaultTheme = OperatingSystem.IsWindows() ? SystemConsoleTheme.Literate : DefaultAnsiTheme; - static readonly TemplateTheme DefaultTemplateTheme = TemplateTheme.Code; - bool _json, _noColor, _forceColor; + public static readonly ConsoleTheme DefaultTheme = + OperatingSystem.IsWindows() ? SystemConsoleTheme.Literate : DefaultAnsiTheme; - public OutputFormatFeature(SeqCliOutputConfig seqCliOutputConfig) - { - _noColor = seqCliOutputConfig.DisableColor; - _forceColor = seqCliOutputConfig.ForceColor; - } + static readonly TemplateTheme DefaultTemplateTheme = Serilog.Templates.Themes.TemplateTheme.Code; public bool Json => _json; @@ -61,20 +81,6 @@ ConsoleTheme Theme : ApplyThemeToRedirectedOutput ? DefaultTemplateTheme : null; - public override void Enable(OptionSet options) - { - options.Add( - "json", - "Print output in newline-delimited JSON (the default is plain text)", - _ => _json = true); - - options.Add("no-color", "Don't colorize text output", _ => _noColor = true); - - options.Add("force-color", - "Force redirected output to have ANSI color (unless `--no-color` is also specified)", - _ => _forceColor = true); - } - public Logger CreateOutputLogger() { var outputConfiguration = new LoggerConfiguration() @@ -98,7 +104,7 @@ public Logger CreateOutputLogger() public void WriteCsv(string csv) { - if (_noColor) + if (_noColor ) { Console.Write(csv); } diff --git a/src/SeqCli/Cli/Features/StoragePathFeature.cs b/src/SeqCli/Cli/Features/StoragePathFeature.cs index 990b09ec..f2f6c2bc 100644 --- a/src/SeqCli/Cli/Features/StoragePathFeature.cs +++ b/src/SeqCli/Cli/Features/StoragePathFeature.cs @@ -29,8 +29,8 @@ public string StorageRootPath public override void Enable(OptionSet options) { options.Add("storage=", - "Set the folder where data will be stored; " + - "" + GetDefaultStorageRoot() + " is used by default.", + "The folder where `SeqCli.json` and other data will be stored; " + + "`" + GetDefaultStorageRoot() + "` is used by default", v => _storageRoot = Path.GetFullPath(v)); } diff --git a/src/SeqCli/Connection/SeqConnectionFactory.cs b/src/SeqCli/Connection/SeqConnectionFactory.cs index 05a82d73..6fdc8322 100644 --- a/src/SeqCli/Connection/SeqConnectionFactory.cs +++ b/src/SeqCli/Connection/SeqConnectionFactory.cs @@ -22,20 +22,13 @@ namespace SeqCli.Connection; class SeqConnectionFactory { - readonly SeqCliConfig _config; - - public SeqConnectionFactory(SeqCliConfig config) - { - _config = config ?? throw new ArgumentNullException(nameof(config)); - } - - public SeqConnection Connect(ConnectionFeature connection) + public SeqConnection Connect(ConnectionFeature connection, SeqCliConfig config) { - var (url, apiKey) = GetConnectionDetails(connection); + var (url, apiKey) = GetConnectionDetails(connection, config); return new SeqConnection(url, apiKey); } - public (string? serverUrl, string? apiKey) GetConnectionDetails(ConnectionFeature connection) + public (string? serverUrl, string? apiKey) GetConnectionDetails(ConnectionFeature connection, SeqCliConfig config) { if (connection == null) throw new ArgumentNullException(nameof(connection)); @@ -47,16 +40,16 @@ public SeqConnection Connect(ConnectionFeature connection) } else if (connection.IsProfileNameSpecified) { - if (!_config.Profiles.TryGetValue(connection.ProfileName!, out var profile)) + if (!config.Profiles.TryGetValue(connection.ProfileName!, out var profile)) throw new ArgumentException($"A profile named `{connection.ProfileName}` was not found; see `seqcli profile list` for available profiles."); url = profile.ServerUrl; - apiKey = profile.DecodeApiKey(_config.Encryption.DataProtector()); + apiKey = profile.DecodeApiKey(config.Encryption.DataProtector()); } else { - url = _config.Connection.ServerUrl; - apiKey = connection.IsApiKeySpecified ? connection.ApiKey : _config.Connection.DecodeApiKey(_config.Encryption.DataProtector()); + url = config.Connection.ServerUrl; + apiKey = connection.IsApiKeySpecified ? connection.ApiKey : config.Connection.DecodeApiKey(config.Encryption.DataProtector()); } return (url, apiKey); diff --git a/src/SeqCli/SeqCliModule.cs b/src/SeqCli/SeqCliModule.cs index 2b04d6f7..7fc0b8f2 100644 --- a/src/SeqCli/SeqCliModule.cs +++ b/src/SeqCli/SeqCliModule.cs @@ -15,10 +15,7 @@ using System.Reflection; using Autofac; using SeqCli.Cli; -using SeqCli.Cli.Features; -using SeqCli.Config; using SeqCli.Connection; -using SeqCli.Encryptor; namespace SeqCli; @@ -26,15 +23,10 @@ class SeqCliModule : Autofac.Module { protected override void Load(ContainerBuilder builder) { - builder.RegisterInstance(new StoragePathFeature()); builder.RegisterType(); builder.RegisterAssemblyTypes(typeof(Program).GetTypeInfo().Assembly) .As() .WithMetadataFrom(); builder.RegisterType(); - builder.Register(c => RuntimeConfigurationLoader.Load(c.Resolve())).SingleInstance(); - builder.Register(c => c.Resolve().Connection).SingleInstance(); - builder.Register(c => c.Resolve().Output).SingleInstance(); - builder.Register(c => c.Resolve().Encryption.DataProtector()).As(); } -} \ No newline at end of file +} From 10db9295281385e0b4413dcc4b4e09fa82c6a86c Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 18 Jul 2025 15:34:34 +1000 Subject: [PATCH 43/51] Make connection factory static --- src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs | 10 +++------- src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/ApiKey/UpdateCommand.cs | 4 ++-- src/SeqCli/Cli/Commands/App/InstallCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/App/ListCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/App/UninstallCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/App/UpdateCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/AppInstance/CreateCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/AppInstance/ListCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/AppInstance/RemoveCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/AppInstance/UpdateCommand.cs | 4 ++-- src/SeqCli/Cli/Commands/Bench/BenchCommand.cs | 8 +++----- src/SeqCli/Cli/Commands/Cluster/HealthCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/Dashboard/ListCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/Dashboard/RemoveCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/Dashboard/RenderCommand.cs | 8 ++------ .../Cli/Commands/ExpressionIndex/CreateCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/ExpressionIndex/ListCommand.cs | 8 ++------ .../Cli/Commands/ExpressionIndex/RemoveCommand.cs | 9 ++------- src/SeqCli/Cli/Commands/Feed/CreateCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/Feed/ListCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/Feed/RemoveCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/Feed/UpdateCommand.cs | 4 ++-- src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs | 8 +++----- src/SeqCli/Cli/Commands/Index/ListCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/Index/SuppressCommand.cs | 7 ++----- src/SeqCli/Cli/Commands/IngestCommand.cs | 8 +++----- src/SeqCli/Cli/Commands/License/ApplyCommand.cs | 7 ++----- src/SeqCli/Cli/Commands/LogCommand.cs | 9 +++------ src/SeqCli/Cli/Commands/Node/HealthCommand.cs | 7 ++----- src/SeqCli/Cli/Commands/Node/ListCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/QueryCommand.cs | 6 ++---- .../Cli/Commands/RetentionPolicy/CreateCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/RetentionPolicy/ListCommand.cs | 8 ++------ .../Cli/Commands/RetentionPolicy/RemoveCommand.cs | 8 ++------ .../Cli/Commands/RetentionPolicy/UpdateCommand.cs | 4 ++-- src/SeqCli/Cli/Commands/Sample/IngestCommand.cs | 9 +++------ src/SeqCli/Cli/Commands/Sample/SetupCommand.cs | 10 +++------- src/SeqCli/Cli/Commands/SearchCommand.cs | 7 ++----- src/SeqCli/Cli/Commands/Settings/ClearCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/Settings/SetCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/Settings/ShowCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/Shared/UpdateCommand.cs | 7 ++----- src/SeqCli/Cli/Commands/Signal/CreateCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/Signal/ImportCommand.cs | 7 ++----- src/SeqCli/Cli/Commands/Signal/ListCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/Signal/RemoveCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/Signal/UpdateCommand.cs | 4 ++-- src/SeqCli/Cli/Commands/TailCommand.cs | 7 ++----- src/SeqCli/Cli/Commands/Template/ExportCommand.cs | 7 ++----- src/SeqCli/Cli/Commands/Template/ImportCommand.cs | 6 ++---- src/SeqCli/Cli/Commands/User/CreateCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/User/ListCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/User/RemoveCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/User/UpdateCommand.cs | 4 ++-- src/SeqCli/Cli/Commands/Workspace/CreateCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/Workspace/ListCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/Workspace/RemoveCommand.cs | 8 ++------ src/SeqCli/Cli/Commands/Workspace/UpdateCommand.cs | 4 ++-- src/SeqCli/Connection/SeqConnectionFactory.cs | 6 +++--- src/SeqCli/SeqCliModule.cs | 1 - 62 files changed, 130 insertions(+), 324 deletions(-) diff --git a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs index 8c5c39c9..514daa0c 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs @@ -32,8 +32,6 @@ namespace SeqCli.Cli.Commands.ApiKey; Example = "seqcli apikey create -t 'Test API Key' -p Environment=Test")] class CreateCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly PropertiesFeature _properties; readonly OutputFormatFeature _output; @@ -43,10 +41,8 @@ class CreateCommand : Command string[]? _permissions; bool _useServerTimestamps, _connectPasswordStdin; - public CreateCommand(SeqConnectionFactory connectionFactory) + public CreateCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "t=|title=", "A title for the API key", @@ -189,13 +185,13 @@ protected override async Task Run() _connectPassword = await Console.In.ReadLineAsync(); } - var (url, _) = _connectionFactory.GetConnectionDetails(_connection, config); + var (url, _) = SeqConnectionFactory.GetConnectionDetails(_connection, config); connection = new SeqConnection(url); await connection.Users.LoginAsync(_connectUsername, _connectPassword ?? ""); } else { - connection = _connectionFactory.Connect(_connection, config); + connection = SeqConnectionFactory.Connect(_connection, config); } return connection; diff --git a/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs index d206fd07..9a7d387a 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs @@ -24,17 +24,13 @@ namespace SeqCli.Cli.Commands.ApiKey; [Command("apikey", "list", "List available API keys", Example="seqcli apikey list")] class ListCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly EntityIdentityFeature _entityIdentity; readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; - public ListCommand(SeqConnectionFactory connectionFactory) + public ListCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - _entityIdentity = Enable(new EntityIdentityFeature("API key", "list")); _output = Enable(); _storagePath = Enable(); @@ -44,7 +40,7 @@ public ListCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var list = _entityIdentity.Id != null ? new[] { await connection.ApiKeys.FindAsync(_entityIdentity.Id) } : diff --git a/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs index 357cbcaf..d809ec7e 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs @@ -26,16 +26,12 @@ namespace SeqCli.Cli.Commands.ApiKey; Example="seqcli apikey remove -t 'Test API Key'")] class RemoveCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly EntityIdentityFeature _entityIdentity; readonly ConnectionFeature _connection; readonly StoragePathFeature _storagePath; - public RemoveCommand(SeqConnectionFactory connectionFactory) + public RemoveCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - _entityIdentity = Enable(new EntityIdentityFeature("API key", "remove")); _connection = Enable(); _storagePath = Enable(); @@ -50,7 +46,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var toRemove = _entityIdentity.Id != null ? [await connection.ApiKeys.FindAsync(_entityIdentity.Id)] : diff --git a/src/SeqCli/Cli/Commands/ApiKey/UpdateCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/UpdateCommand.cs index b8ecff35..46a77309 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/UpdateCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/UpdateCommand.cs @@ -20,6 +20,6 @@ namespace SeqCli.Cli.Commands.ApiKey; [Command("apikey", "update", "Update an existing API key", Example="seqcli apikey update --json '{...}'")] -class UpdateCommand(SeqConnectionFactory connectionFactory): - Shared.UpdateCommand(connectionFactory, "apikey", nameof(SeqConnection.ApiKeys), "API key"); +class UpdateCommand(): + Shared.UpdateCommand("apikey", nameof(SeqConnection.ApiKeys), "API key"); \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/App/InstallCommand.cs b/src/SeqCli/Cli/Commands/App/InstallCommand.cs index 76a57cf7..4d284b29 100644 --- a/src/SeqCli/Cli/Commands/App/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/App/InstallCommand.cs @@ -29,18 +29,14 @@ namespace SeqCli.Cli.Commands.App; // ReSharper disable once UnusedType.Global class InstallCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; string? _packageId, _version, _feedId; - public InstallCommand(SeqConnectionFactory connectionFactory) + public InstallCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "package-id=", "The package id of the app to install", @@ -70,7 +66,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var feedId = _feedId; if (feedId == null) diff --git a/src/SeqCli/Cli/Commands/App/ListCommand.cs b/src/SeqCli/Cli/Commands/App/ListCommand.cs index d0b1b41f..166f3bc5 100644 --- a/src/SeqCli/Cli/Commands/App/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/App/ListCommand.cs @@ -10,8 +10,6 @@ namespace SeqCli.Cli.Commands.App; [Command("app", "list", "List installed app packages", Example="seqcli app list")] class ListCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - string? _title, _id; readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; @@ -20,10 +18,8 @@ class ListCommand : Command string? PackageId => string.IsNullOrWhiteSpace(_title) ? null : _title.Trim(); string? Id => string.IsNullOrWhiteSpace(_id) ? null : _id.Trim(); - public ListCommand(SeqConnectionFactory connectionFactory) + public ListCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "package-id=", "The package id of the app(s) to list", @@ -48,7 +44,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var list = Id != null ? [await connection.Apps.FindAsync(Id)] : diff --git a/src/SeqCli/Cli/Commands/App/UninstallCommand.cs b/src/SeqCli/Cli/Commands/App/UninstallCommand.cs index 5423c0ac..03f40b02 100644 --- a/src/SeqCli/Cli/Commands/App/UninstallCommand.cs +++ b/src/SeqCli/Cli/Commands/App/UninstallCommand.cs @@ -14,16 +14,12 @@ namespace SeqCli.Cli.Commands.App; // ReSharper disable once UnusedType.Global class UninstallCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - string? _packageId, _id; readonly ConnectionFeature _connection; readonly StoragePathFeature _storagePath; - public UninstallCommand(SeqConnectionFactory connectionFactory) + public UninstallCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "package-id=", "The package id of the app package to uninstall", @@ -47,7 +43,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var toRemove = _id != null ? [await connection.Apps.FindAsync(_id)] : (await connection.Apps.ListAsync()) diff --git a/src/SeqCli/Cli/Commands/App/UpdateCommand.cs b/src/SeqCli/Cli/Commands/App/UpdateCommand.cs index 429c1977..4c30d62b 100644 --- a/src/SeqCli/Cli/Commands/App/UpdateCommand.cs +++ b/src/SeqCli/Cli/Commands/App/UpdateCommand.cs @@ -28,8 +28,6 @@ namespace SeqCli.Cli.Commands.App; // ReSharper disable once UnusedType.Global class UpdateCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; @@ -37,10 +35,8 @@ class UpdateCommand : Command string? _id, _name, _version; bool _all, _force; - public UpdateCommand(SeqConnectionFactory connectionFactory) + public UpdateCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "i=|id=", "The id of a single installed app to update", @@ -94,7 +90,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var output = _output.GetOutputFormat(config); var apps = await connection.Apps.ListAsync(); diff --git a/src/SeqCli/Cli/Commands/AppInstance/CreateCommand.cs b/src/SeqCli/Cli/Commands/AppInstance/CreateCommand.cs index 56b2e03d..af5aa731 100644 --- a/src/SeqCli/Cli/Commands/AppInstance/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/AppInstance/CreateCommand.cs @@ -16,8 +16,6 @@ namespace SeqCli.Cli.Commands.AppInstance; Example = "seqcli appinstance create -t 'Email Ops' --app hostedapp-314159 -p To=ops@example.com")] class CreateCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; @@ -27,10 +25,8 @@ class CreateCommand : Command readonly List _overridable = new(); bool _streamIncomingEvents; - public CreateCommand(SeqConnectionFactory connectionFactory) + public CreateCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "t=|title=", "A title for the app instance", @@ -78,7 +74,7 @@ public CreateCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); AppInstanceEntity instance = await connection.AppInstances.TemplateAsync(_appId)!; diff --git a/src/SeqCli/Cli/Commands/AppInstance/ListCommand.cs b/src/SeqCli/Cli/Commands/AppInstance/ListCommand.cs index 2d67736d..2e9e2c45 100644 --- a/src/SeqCli/Cli/Commands/AppInstance/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/AppInstance/ListCommand.cs @@ -10,17 +10,13 @@ namespace SeqCli.Cli.Commands.AppInstance; [Command("appinstance", "list", "List instances of installed apps", Example="seqcli appinstance list")] class ListCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly EntityIdentityFeature _entityIdentity; readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; - public ListCommand(SeqConnectionFactory connectionFactory) + public ListCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - _entityIdentity = Enable(new EntityIdentityFeature("app instance", "list")); _output = Enable(); _storagePath = Enable(); @@ -30,7 +26,7 @@ public ListCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var list = _entityIdentity.Id != null ? [await connection.AppInstances.FindAsync(_entityIdentity.Id)] : diff --git a/src/SeqCli/Cli/Commands/AppInstance/RemoveCommand.cs b/src/SeqCli/Cli/Commands/AppInstance/RemoveCommand.cs index a87f0d72..33da86b2 100644 --- a/src/SeqCli/Cli/Commands/AppInstance/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/AppInstance/RemoveCommand.cs @@ -12,16 +12,12 @@ namespace SeqCli.Cli.Commands.AppInstance; class RemoveCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly EntityIdentityFeature _entityIdentity; readonly ConnectionFeature _connection; readonly StoragePathFeature _storagePath; - public RemoveCommand(SeqConnectionFactory connectionFactory) + public RemoveCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - _entityIdentity = Enable(new EntityIdentityFeature("app instance", "remove")); _connection = Enable(); _storagePath = Enable(); @@ -36,7 +32,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var toRemove = _entityIdentity.Id != null ? [await connection.AppInstances.FindAsync(_entityIdentity.Id)] : diff --git a/src/SeqCli/Cli/Commands/AppInstance/UpdateCommand.cs b/src/SeqCli/Cli/Commands/AppInstance/UpdateCommand.cs index f46d7760..22e44af1 100644 --- a/src/SeqCli/Cli/Commands/AppInstance/UpdateCommand.cs +++ b/src/SeqCli/Cli/Commands/AppInstance/UpdateCommand.cs @@ -20,6 +20,6 @@ namespace SeqCli.Cli.Commands.AppInstance; [Command("appinstance", "update", "Update an existing app instance", Example="seqcli appinstance update --json '{...}'")] -class UpdateCommand(SeqConnectionFactory connectionFactory): - Shared.UpdateCommand(connectionFactory, "appinstance", nameof(SeqConnection.AppInstances), "app instance"); +class UpdateCommand(): + Shared.UpdateCommand("appinstance", nameof(SeqConnection.AppInstances), "app instance"); \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Bench/BenchCommand.cs b/src/SeqCli/Cli/Commands/Bench/BenchCommand.cs index fab25a41..596bbbe4 100644 --- a/src/SeqCli/Cli/Commands/Bench/BenchCommand.cs +++ b/src/SeqCli/Cli/Commands/Bench/BenchCommand.cs @@ -66,7 +66,6 @@ namespace SeqCli.Cli.Commands.Bench; [Command("bench", @"Measure query performance")] class BenchCommand : Command { - readonly SeqConnectionFactory _connectionFactory; int _runs = 10; readonly ConnectionFeature _connection; readonly DateRangeFeature _range; @@ -79,9 +78,8 @@ class BenchCommand : Command bool _withIngestion = false; bool _withQueries = false; - public BenchCommand(SeqConnectionFactory connectionFactory) + public BenchCommand() { - _connectionFactory = connectionFactory; Options.Add("r|runs=", "The number of runs to execute; the default is 10", r => _runs = int.Parse(r)); Options.Add( @@ -128,8 +126,8 @@ protected override async Task Run() try { var config = RuntimeConfigurationLoader.Load(_storagePath); - var (_, apiKey) = _connectionFactory.GetConnectionDetails(_connection, config); - var connection = _connectionFactory.Connect(_connection, config); + var (_, apiKey) = SeqConnectionFactory.GetConnectionDetails(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var timeout = _timeout.ApplyTimeout(connection.Client.HttpClient); var seqVersion = (await connection.Client.GetRootAsync()).Version; await using var reportingLogger = BuildReportingLogger(); diff --git a/src/SeqCli/Cli/Commands/Cluster/HealthCommand.cs b/src/SeqCli/Cli/Commands/Cluster/HealthCommand.cs index e3daa609..a868cb52 100644 --- a/src/SeqCli/Cli/Commands/Cluster/HealthCommand.cs +++ b/src/SeqCli/Cli/Commands/Cluster/HealthCommand.cs @@ -31,18 +31,14 @@ namespace SeqCli.Cli.Commands.Cluster; Example = "seqcli cluster health -s https://seq.example.com --wait-until-healthy")] class HealthCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly TimeoutFeature _timeout; readonly WaitUntilHealthyFeature _waitUntilHealthy; readonly StoragePathFeature _storagePath; - public HealthCommand(SeqConnectionFactory connectionFactory) + public HealthCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - _waitUntilHealthy = Enable(new WaitUntilHealthyFeature("cluster")); _timeout = Enable(new TimeoutFeature()); _output = Enable(); @@ -53,7 +49,7 @@ public HealthCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var timeout = _timeout.ApplyTimeout(connection.Client.HttpClient); diff --git a/src/SeqCli/Cli/Commands/Dashboard/ListCommand.cs b/src/SeqCli/Cli/Commands/Dashboard/ListCommand.cs index 357fc99c..087f9fc4 100644 --- a/src/SeqCli/Cli/Commands/Dashboard/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Dashboard/ListCommand.cs @@ -24,18 +24,14 @@ namespace SeqCli.Cli.Commands.Dashboard; [Command("dashboard", "list", "List dashboards", Example="seqcli dashboard list")] class ListCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly EntityIdentityFeature _entityIdentity; readonly EntityOwnerFeature _entityOwner; readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; - public ListCommand(SeqConnectionFactory connectionFactory) + public ListCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - _entityIdentity = Enable(new EntityIdentityFeature("dashboard", "list")); _entityOwner = Enable(new EntityOwnerFeature("dashboard", "list", "listed", _entityIdentity)); _output = Enable(); @@ -46,7 +42,7 @@ public ListCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var list = _entityIdentity.Id != null ? [await connection.Dashboards.FindAsync(_entityIdentity.Id)] : diff --git a/src/SeqCli/Cli/Commands/Dashboard/RemoveCommand.cs b/src/SeqCli/Cli/Commands/Dashboard/RemoveCommand.cs index b039180c..61022b26 100644 --- a/src/SeqCli/Cli/Commands/Dashboard/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/Dashboard/RemoveCommand.cs @@ -26,17 +26,13 @@ namespace SeqCli.Cli.Commands.Dashboard; Example="seqcli dashboard remove -i dashboard-159")] class RemoveCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly EntityIdentityFeature _entityIdentity; readonly EntityOwnerFeature _entityOwner; readonly ConnectionFeature _connection; readonly StoragePathFeature _storagePath; - public RemoveCommand(SeqConnectionFactory connectionFactory) + public RemoveCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - _entityIdentity = Enable(new EntityIdentityFeature("dashboard", "remove")); _entityOwner = Enable(new EntityOwnerFeature("dashboard", "remove", "removed", _entityIdentity)); _connection = Enable(); @@ -52,7 +48,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var toRemove = _entityIdentity.Id != null ? [await connection.Dashboards.FindAsync(_entityIdentity.Id)] : diff --git a/src/SeqCli/Cli/Commands/Dashboard/RenderCommand.cs b/src/SeqCli/Cli/Commands/Dashboard/RenderCommand.cs index 683efc0d..652066a0 100644 --- a/src/SeqCli/Cli/Commands/Dashboard/RenderCommand.cs +++ b/src/SeqCli/Cli/Commands/Dashboard/RenderCommand.cs @@ -33,8 +33,6 @@ class RenderCommand : Command { const int MaximumReturnedHitRows = 10000; - readonly SeqConnectionFactory _connectionFactory; - readonly DateRangeFeature _range; readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; @@ -44,10 +42,8 @@ class RenderCommand : Command string? _id, _lastDuration, _intervalDuration, _chartTitle; - public RenderCommand(SeqConnectionFactory connectionFactory) + public RenderCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "i=|id=", "The id of a single dashboard to render", @@ -71,7 +67,7 @@ public RenderCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); if (_id == null) { diff --git a/src/SeqCli/Cli/Commands/ExpressionIndex/CreateCommand.cs b/src/SeqCli/Cli/Commands/ExpressionIndex/CreateCommand.cs index d2b0d912..d0760376 100644 --- a/src/SeqCli/Cli/Commands/ExpressionIndex/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/ExpressionIndex/CreateCommand.cs @@ -29,18 +29,14 @@ namespace SeqCli.Cli.Commands.ExpressionIndex; Example = "seqcli expressionindex create --expression \"ServerName\"")] class CreateCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; string? _expression; - public CreateCommand(SeqConnectionFactory connectionFactory) + public CreateCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "e=|expression=", "The expression to index", @@ -54,7 +50,7 @@ public CreateCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); if (string.IsNullOrEmpty(_expression)) { diff --git a/src/SeqCli/Cli/Commands/ExpressionIndex/ListCommand.cs b/src/SeqCli/Cli/Commands/ExpressionIndex/ListCommand.cs index 6a8656af..16b00cb8 100644 --- a/src/SeqCli/Cli/Commands/ExpressionIndex/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/ExpressionIndex/ListCommand.cs @@ -9,18 +9,14 @@ namespace SeqCli.Cli.Commands.ExpressionIndex; [Command("expressionindex", "list", "List expression indexes", Example="seqcli expressionindex list")] class ListCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; string? _id; - public ListCommand(SeqConnectionFactory connectionFactory) + public ListCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "i=|id=", "The id of a single expression index to list", @@ -34,7 +30,7 @@ public ListCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var list = _id is not null ? [await connection.ExpressionIndexes.FindAsync(_id)] : await connection.ExpressionIndexes.ListAsync(); diff --git a/src/SeqCli/Cli/Commands/ExpressionIndex/RemoveCommand.cs b/src/SeqCli/Cli/Commands/ExpressionIndex/RemoveCommand.cs index 21abc415..b00e7a41 100644 --- a/src/SeqCli/Cli/Commands/ExpressionIndex/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/ExpressionIndex/RemoveCommand.cs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; using System.Threading.Tasks; using SeqCli.Cli.Features; using SeqCli.Config; @@ -25,16 +24,12 @@ namespace SeqCli.Cli.Commands.ExpressionIndex; Example = "seqcli expressionindex -i expressionindex-2529")] class RemoveCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly StoragePathFeature _storagePath; string? _id; - public RemoveCommand(SeqConnectionFactory connectionFactory) + public RemoveCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "i=|id=", "The id of an expression index to remove", @@ -53,7 +48,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var toRemove = await connection.ExpressionIndexes.FindAsync(_id); await connection.ExpressionIndexes.RemoveAsync(toRemove); diff --git a/src/SeqCli/Cli/Commands/Feed/CreateCommand.cs b/src/SeqCli/Cli/Commands/Feed/CreateCommand.cs index e7750cac..9d4aea80 100644 --- a/src/SeqCli/Cli/Commands/Feed/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Feed/CreateCommand.cs @@ -26,8 +26,6 @@ namespace SeqCli.Cli.Commands.Feed; Example = "seqcli feed create -n 'CI' --location=\"https://f.feedz.io/example/ci\" -u Seq --password-stdin")] class CreateCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; @@ -35,10 +33,8 @@ class CreateCommand : Command string? _name, _location, _username, _password; bool _passwordStdin; - public CreateCommand(SeqConnectionFactory connectionFactory) + public CreateCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "n=|name=", "A unique name for the feed", @@ -72,7 +68,7 @@ public CreateCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var feed = await connection.Feeds.TemplateAsync(); feed.Name = _name; diff --git a/src/SeqCli/Cli/Commands/Feed/ListCommand.cs b/src/SeqCli/Cli/Commands/Feed/ListCommand.cs index 42c06565..a7d45220 100644 --- a/src/SeqCli/Cli/Commands/Feed/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Feed/ListCommand.cs @@ -24,18 +24,14 @@ namespace SeqCli.Cli.Commands.Feed; [Command("feed", "list", "List NuGet feeds", Example="seqcli feed list")] class ListCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; string? _name, _id; - public ListCommand(SeqConnectionFactory connectionFactory) + public ListCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "n=|name=", "The name of the feed to list", @@ -54,7 +50,7 @@ public ListCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var list = _id != null ? [await connection.Feeds.FindAsync(_id)] : diff --git a/src/SeqCli/Cli/Commands/Feed/RemoveCommand.cs b/src/SeqCli/Cli/Commands/Feed/RemoveCommand.cs index 097c6d48..3180bfb5 100644 --- a/src/SeqCli/Cli/Commands/Feed/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/Feed/RemoveCommand.cs @@ -26,17 +26,13 @@ namespace SeqCli.Cli.Commands.Feed; Example="seqcli feed remove -n CI")] class RemoveCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly StoragePathFeature _storagePath; string? _name, _id; - public RemoveCommand(SeqConnectionFactory connectionFactory) + public RemoveCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "n=|name=", "The name of the feed to remove", @@ -60,7 +56,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var toRemove = _id != null ? [await connection.Feeds.FindAsync(_id)] : diff --git a/src/SeqCli/Cli/Commands/Feed/UpdateCommand.cs b/src/SeqCli/Cli/Commands/Feed/UpdateCommand.cs index 1dd5d265..59ed98b3 100644 --- a/src/SeqCli/Cli/Commands/Feed/UpdateCommand.cs +++ b/src/SeqCli/Cli/Commands/Feed/UpdateCommand.cs @@ -20,6 +20,6 @@ namespace SeqCli.Cli.Commands.Feed; [Command("feed", "update", "Update an existing NuGet feed", Example="seqcli feed update --json '{...}'")] -class UpdateCommand(SeqConnectionFactory connectionFactory): - Shared.UpdateCommand(connectionFactory, "feed", nameof(SeqConnection.Feeds), "NuGet feed"); +class UpdateCommand(): + Shared.UpdateCommand("feed", nameof(SeqConnection.Feeds), "NuGet feed"); \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index 7efe2932..f1bfc7f5 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -48,16 +48,14 @@ namespace SeqCli.Cli.Commands.Forwarder; [Command("forwarder", "run", "Listen on an HTTP endpoint and forward ingested logs to Seq", IsPreview = true)] class RunCommand : Command { - readonly SeqConnectionFactory _connectionFactory; readonly StoragePathFeature _storagePath; readonly ListenUriFeature _listenUri; readonly ConnectionFeature _connection; bool _noLogo; - public RunCommand(SeqConnectionFactory connectionFactory) + public RunCommand() { - _connectionFactory = connectionFactory; Options.Add("nologo", _ => _noLogo = true); _listenUri = Enable(); _connection = Enable(); @@ -93,12 +91,12 @@ protected override async Task Run(string[] unrecognized) return 1; } - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); // The API key is passed through separately because `SeqConnection` doesn't expose a batched ingestion // mechanism and so we manually construct `HttpRequestMessage`s deeper in the stack. Nice feature gap to // close at some point! - var (serverUrl, apiKey) = _connectionFactory.GetConnectionDetails(_connection, config); + var (serverUrl, apiKey) = SeqConnectionFactory.GetConnectionDetails(_connection, config); Log.Logger = CreateLogger( config.Forwarder.Diagnostics.InternalLoggingLevel, diff --git a/src/SeqCli/Cli/Commands/Index/ListCommand.cs b/src/SeqCli/Cli/Commands/Index/ListCommand.cs index b557f5fd..78138f70 100644 --- a/src/SeqCli/Cli/Commands/Index/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Index/ListCommand.cs @@ -26,18 +26,14 @@ namespace SeqCli.Cli.Commands.Index; [Command("index", "list", "List indexes", Example="seqcli index list")] class ListCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; string? _id; - public ListCommand(SeqConnectionFactory connectionFactory) + public ListCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "i=|id=", "The id of a single index to list", @@ -51,7 +47,7 @@ public ListCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var list = _id is not null ? [await connection.Indexes.FindAsync(_id)] diff --git a/src/SeqCli/Cli/Commands/Index/SuppressCommand.cs b/src/SeqCli/Cli/Commands/Index/SuppressCommand.cs index fb00d52e..bd330179 100644 --- a/src/SeqCli/Cli/Commands/Index/SuppressCommand.cs +++ b/src/SeqCli/Cli/Commands/Index/SuppressCommand.cs @@ -25,16 +25,13 @@ namespace SeqCli.Cli.Commands.Index; [Command("index", "suppress", "Suppress an index", Example="seqcli index suppress -i index-2191448f1d9b4f22bd32c6edef752748")] class SuppressCommand : Command { - readonly SeqConnectionFactory _connectionFactory; readonly ConnectionFeature _connection; readonly StoragePathFeature _storagePath; string? _id; - public SuppressCommand(SeqConnectionFactory connectionFactory) + public SuppressCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "i=|id=", "The id of an index to suppress", @@ -53,7 +50,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var toSuppress = await connection.Indexes.FindAsync(_id); await connection.Indexes.SuppressAsync(toSuppress); diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index 8109fcfe..1827ed59 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -34,7 +34,6 @@ class IngestCommand : Command { const string DefaultPattern = "{@m:line}"; - readonly SeqConnectionFactory _connectionFactory; readonly InvalidDataHandlingFeature _invalidDataHandlingFeature; readonly FileInputFeature _fileInputFeature; readonly PropertiesFeature _properties; @@ -46,9 +45,8 @@ class IngestCommand : Command string _pattern = DefaultPattern; bool _json; - public IngestCommand(SeqConnectionFactory connectionFactory) + public IngestCommand() { - _connectionFactory = connectionFactory; _fileInputFeature = Enable(new FileInputFeature("File(s) to ingest", allowMultiple: true)); _invalidDataHandlingFeature = Enable(); _properties = Enable(); @@ -101,12 +99,12 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); // The API key is passed through separately because `SeqConnection` doesn't expose a batched ingestion // mechanism and so we manually construct `HttpRequestMessage`s deeper in the stack. Nice feature gap to // close at some point! - var (_, apiKey) = _connectionFactory.GetConnectionDetails(_connection, config); + var (_, apiKey) = SeqConnectionFactory.GetConnectionDetails(_connection, config); var batchSize = _batchSize.Value; foreach (var input in _fileInputFeature.OpenInputs()) diff --git a/src/SeqCli/Cli/Commands/License/ApplyCommand.cs b/src/SeqCli/Cli/Commands/License/ApplyCommand.cs index 28ac4585..53b50da9 100644 --- a/src/SeqCli/Cli/Commands/License/ApplyCommand.cs +++ b/src/SeqCli/Cli/Commands/License/ApplyCommand.cs @@ -16,7 +16,6 @@ namespace SeqCli.Cli.Commands.License; Example = "seqcli license apply --certificate=\"license.txt\"")] class ApplyCommand : Command { - readonly SeqConnectionFactory _connectionFactory; readonly ConnectionFeature _connection; readonly StoragePathFeature _storagePath; @@ -24,10 +23,8 @@ class ApplyCommand : Command bool _certificateStdin; bool _automaticallyRefresh; - public ApplyCommand(SeqConnectionFactory connectionFactory) + public ApplyCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add("c=|certificate=", "Certificate file; the file must be UTF-8 text", v => _certificateFilename = ArgumentString.Normalize(v)); @@ -75,7 +72,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var license = await connection.Licenses.FindCurrentAsync(); license.LicenseText = certificate; license.AutomaticallyRefresh = _automaticallyRefresh; diff --git a/src/SeqCli/Cli/Commands/LogCommand.cs b/src/SeqCli/Cli/Commands/LogCommand.cs index a8f0df05..be3c5f8e 100644 --- a/src/SeqCli/Cli/Commands/LogCommand.cs +++ b/src/SeqCli/Cli/Commands/LogCommand.cs @@ -33,16 +33,13 @@ namespace SeqCli.Cli.Commands; [Command("log", "Send a structured log event to the server", Example = "seqcli log -m 'Hello, {Name}!' -p Name=World -p App=Test")] class LogCommand : Command { - readonly SeqConnectionFactory _connectionFactory; readonly PropertiesFeature _properties; readonly ConnectionFeature _connection; readonly StoragePathFeature _storagePath; string? _message, _level, _timestamp, _exception; - public LogCommand(SeqConnectionFactory connectionFactory) + public LogCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "m=|message=", "A message to associate with the event (the default is to send no message); https://messagetemplates.org syntax is supported", @@ -107,8 +104,8 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); - var (_, apiKey) = _connectionFactory.GetConnectionDetails(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); + var (_, apiKey) = SeqConnectionFactory.GetConnectionDetails(_connection, config); var request = new HttpRequestMessage(HttpMethod.Post, ApiConstants.IngestionEndpoint) {Content = content}; if (apiKey != null) diff --git a/src/SeqCli/Cli/Commands/Node/HealthCommand.cs b/src/SeqCli/Cli/Commands/Node/HealthCommand.cs index 0e1ce0a5..60a82b76 100644 --- a/src/SeqCli/Cli/Commands/Node/HealthCommand.cs +++ b/src/SeqCli/Cli/Commands/Node/HealthCommand.cs @@ -31,17 +31,14 @@ namespace SeqCli.Cli.Commands.Node; Example = "seqcli node health -s https://seq-2.example.com")] class HealthCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly WaitUntilHealthyFeature _waitUntilHealthy; readonly TimeoutFeature _timeout; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; - public HealthCommand(SeqConnectionFactory connectionFactory) + public HealthCommand() { - _connectionFactory = connectionFactory; _waitUntilHealthy = Enable(new WaitUntilHealthyFeature("node")); _timeout = Enable(new TimeoutFeature()); @@ -53,7 +50,7 @@ public HealthCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var timeout = _timeout.ApplyTimeout(connection.Client.HttpClient); diff --git a/src/SeqCli/Cli/Commands/Node/ListCommand.cs b/src/SeqCli/Cli/Commands/Node/ListCommand.cs index 10f9cc8d..c374c6a0 100644 --- a/src/SeqCli/Cli/Commands/Node/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Node/ListCommand.cs @@ -25,18 +25,14 @@ namespace SeqCli.Cli.Commands.Node; Example = "seqcli node list --json")] class ListCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; string? _name, _id; - public ListCommand(SeqConnectionFactory connectionFactory) + public ListCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "n=|name=", "The name of the cluster node to list", @@ -55,7 +51,7 @@ public ListCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var list = _id != null ? [await connection.Cluster.FindAsync(_id)] : diff --git a/src/SeqCli/Cli/Commands/QueryCommand.cs b/src/SeqCli/Cli/Commands/QueryCommand.cs index 3cafab0f..1c54110f 100644 --- a/src/SeqCli/Cli/Commands/QueryCommand.cs +++ b/src/SeqCli/Cli/Commands/QueryCommand.cs @@ -29,7 +29,6 @@ namespace SeqCli.Cli.Commands; class QueryCommand : Command { readonly OutputFormatFeature _output; - readonly SeqConnectionFactory _connectionFactory; readonly ConnectionFeature _connection; readonly DateRangeFeature _range; readonly SignalExpressionFeature _signal; @@ -38,9 +37,8 @@ class QueryCommand : Command string? _query; bool _trace; - public QueryCommand(SeqConnectionFactory connectionFactory) + public QueryCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); Options.Add("q=|query=", "The query to execute", v => _query = v); _range = Enable(); _signal = Enable(); @@ -60,7 +58,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var timeout = _timeout.ApplyTimeout(connection.Client.HttpClient); diff --git a/src/SeqCli/Cli/Commands/RetentionPolicy/CreateCommand.cs b/src/SeqCli/Cli/Commands/RetentionPolicy/CreateCommand.cs index 5cd947c0..aa5c6a07 100644 --- a/src/SeqCli/Cli/Commands/RetentionPolicy/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/RetentionPolicy/CreateCommand.cs @@ -29,8 +29,6 @@ namespace SeqCli.Cli.Commands.RetentionPolicy; Example = "seqcli retention create --after 30d --delete-all-events")] class CreateCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; @@ -39,10 +37,8 @@ class CreateCommand : Command bool _deleteAllEvents; string? _deleteMatchingSignal; - public CreateCommand(SeqConnectionFactory connectionFactory) + public CreateCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "after=", "A duration after which the policy will delete events, e.g. `7d`", @@ -70,7 +66,7 @@ public CreateCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); SignalExpressionPart? removedSignalExpression; diff --git a/src/SeqCli/Cli/Commands/RetentionPolicy/ListCommand.cs b/src/SeqCli/Cli/Commands/RetentionPolicy/ListCommand.cs index b9a45e1c..391866ad 100644 --- a/src/SeqCli/Cli/Commands/RetentionPolicy/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/RetentionPolicy/ListCommand.cs @@ -23,18 +23,14 @@ namespace SeqCli.Cli.Commands.RetentionPolicy; [Command("retention", "list", "List retention policies", Example="seqcli retention list")] class ListCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; string? _id; - public ListCommand(SeqConnectionFactory connectionFactory) + public ListCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "i=|id=", "The id of a single retention policy to list", @@ -48,7 +44,7 @@ public ListCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var list = _id != null ? [await connection.RetentionPolicies.FindAsync(_id)] : diff --git a/src/SeqCli/Cli/Commands/RetentionPolicy/RemoveCommand.cs b/src/SeqCli/Cli/Commands/RetentionPolicy/RemoveCommand.cs index 2fa961f9..3d9266e0 100644 --- a/src/SeqCli/Cli/Commands/RetentionPolicy/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/RetentionPolicy/RemoveCommand.cs @@ -25,17 +25,13 @@ namespace SeqCli.Cli.Commands.RetentionPolicy; Example="seqcli retention remove -i retentionpolicy-17")] class RemoveCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly StoragePathFeature _storagePath; string? _id; - public RemoveCommand(SeqConnectionFactory connectionFactory) + public RemoveCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "i=|id=", "The id of a single retention policy to remove", @@ -54,7 +50,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var toRemove = await connection.RetentionPolicies.FindAsync(_id); diff --git a/src/SeqCli/Cli/Commands/RetentionPolicy/UpdateCommand.cs b/src/SeqCli/Cli/Commands/RetentionPolicy/UpdateCommand.cs index 12dded9e..ca6503ce 100644 --- a/src/SeqCli/Cli/Commands/RetentionPolicy/UpdateCommand.cs +++ b/src/SeqCli/Cli/Commands/RetentionPolicy/UpdateCommand.cs @@ -20,6 +20,6 @@ namespace SeqCli.Cli.Commands.RetentionPolicy; [Command("retention", "update", "Update an existing retention policy", Example="seqcli retention update --json '{...}'")] -class UpdateCommand(SeqConnectionFactory connectionFactory): - Shared.UpdateCommand(connectionFactory, "retention", nameof(SeqConnection.RetentionPolicies), "retention policy"); +class UpdateCommand(): + Shared.UpdateCommand("retention", nameof(SeqConnection.RetentionPolicies), "retention policy"); \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Sample/IngestCommand.cs b/src/SeqCli/Cli/Commands/Sample/IngestCommand.cs index b4cbd2b7..a5dd3790 100644 --- a/src/SeqCli/Cli/Commands/Sample/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/Sample/IngestCommand.cs @@ -26,8 +26,6 @@ namespace SeqCli.Cli.Commands.Sample; Example = "seqcli sample ingest")] class IngestCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly ConfirmFeature _confirm; readonly BatchSizeFeature _batchSize; @@ -37,9 +35,8 @@ class IngestCommand : Command bool _setup; int _simulations = 1; - public IngestCommand(SeqConnectionFactory connectionFactory) + public IngestCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); _confirm = Enable(); _connection = Enable(); @@ -55,7 +52,7 @@ public IngestCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var (url, apiKey) = _connectionFactory.GetConnectionDetails(_connection, config); + var (url, apiKey) = SeqConnectionFactory.GetConnectionDetails(_connection, config); var batchSize = _batchSize.Value; if (!_confirm.TryConfirm(_setup @@ -66,7 +63,7 @@ protected override async Task Run() return 1; } - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); if (_setup) { diff --git a/src/SeqCli/Cli/Commands/Sample/SetupCommand.cs b/src/SeqCli/Cli/Commands/Sample/SetupCommand.cs index 0f5dadea..fb0c0722 100644 --- a/src/SeqCli/Cli/Commands/Sample/SetupCommand.cs +++ b/src/SeqCli/Cli/Commands/Sample/SetupCommand.cs @@ -33,16 +33,12 @@ namespace SeqCli.Cli.Commands.Sample; Example = "seqcli sample setup")] class SetupCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly ConfirmFeature _confirm; readonly StoragePathFeature _storagePath; - public SetupCommand(SeqConnectionFactory connectionFactory) + public SetupCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - // The command will also at some point accept an `--allow-outbound-requests` flag, which will cause sample // apps to be installed, and a health check to be set up. @@ -54,9 +50,9 @@ public SetupCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); - var (url, _) = _connectionFactory.GetConnectionDetails(_connection, config); + var (url, _) = SeqConnectionFactory.GetConnectionDetails(_connection, config); if (!_confirm.TryConfirm($"This will apply sample configuration items to the Seq server at {url}.")) { await Console.Error.WriteLineAsync("Canceled by user."); diff --git a/src/SeqCli/Cli/Commands/SearchCommand.cs b/src/SeqCli/Cli/Commands/SearchCommand.cs index 016d36c2..151be6eb 100644 --- a/src/SeqCli/Cli/Commands/SearchCommand.cs +++ b/src/SeqCli/Cli/Commands/SearchCommand.cs @@ -34,7 +34,6 @@ namespace SeqCli.Cli.Commands; Example = "seqcli search -f \"@Exception like '%TimeoutException%'\" -c 30")] class SearchCommand : Command { - readonly SeqConnectionFactory _connectionFactory; readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly DateRangeFeature _range; @@ -45,10 +44,8 @@ class SearchCommand : Command int _httpClientTimeout = 100000; bool _trace, _noWebSockets; - public SearchCommand(SeqConnectionFactory connectionFactory) + public SearchCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "f=|filter=", "A filter to apply to the search, for example `Host = 'xmpweb-01.example.com'`", @@ -81,7 +78,7 @@ protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); await using var output = _output.GetOutputFormat(config).CreateOutputLogger(); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); connection.Client.HttpClient.Timeout = TimeSpan.FromMilliseconds(_httpClientTimeout); string? filter = null; diff --git a/src/SeqCli/Cli/Commands/Settings/ClearCommand.cs b/src/SeqCli/Cli/Commands/Settings/ClearCommand.cs index 08d79ba1..7d527e43 100644 --- a/src/SeqCli/Cli/Commands/Settings/ClearCommand.cs +++ b/src/SeqCli/Cli/Commands/Settings/ClearCommand.cs @@ -23,16 +23,12 @@ namespace SeqCli.Cli.Commands.Settings; [Command("setting", "clear", "Clear a runtime-configurable server setting")] class ClearCommand: Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly SettingNameFeature _name; readonly StoragePathFeature _storagePath; - public ClearCommand(SeqConnectionFactory connectionFactory) + public ClearCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - _name = Enable(); _connection = Enable(); _storagePath = Enable(); @@ -41,7 +37,7 @@ public ClearCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var setting = await connection.Settings.FindNamedAsync(_name.Name); setting.Value = null; diff --git a/src/SeqCli/Cli/Commands/Settings/SetCommand.cs b/src/SeqCli/Cli/Commands/Settings/SetCommand.cs index 4c3f3051..edc26154 100644 --- a/src/SeqCli/Cli/Commands/Settings/SetCommand.cs +++ b/src/SeqCli/Cli/Commands/Settings/SetCommand.cs @@ -24,8 +24,6 @@ namespace SeqCli.Cli.Commands.Settings; [Command("setting", "set", "Change a runtime-configurable server setting")] class SetCommand: Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly SettingNameFeature _name; readonly StoragePathFeature _storagePath; @@ -33,10 +31,8 @@ class SetCommand: Command string? _value; bool _valueSpecified, _readValueFromStdin; - public SetCommand(SeqConnectionFactory connectionFactory) + public SetCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - _name = Enable(); Options.Add("v|value=", @@ -65,7 +61,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var setting = await connection.Settings.FindNamedAsync(_name.Name); setting.Value = ReadValue(); diff --git a/src/SeqCli/Cli/Commands/Settings/ShowCommand.cs b/src/SeqCli/Cli/Commands/Settings/ShowCommand.cs index 93d1bd9e..f6f7ef5b 100644 --- a/src/SeqCli/Cli/Commands/Settings/ShowCommand.cs +++ b/src/SeqCli/Cli/Commands/Settings/ShowCommand.cs @@ -24,16 +24,12 @@ namespace SeqCli.Cli.Commands.Settings; [Command("setting", "show", "Print the current value of a runtime-configurable server setting")] class ShowCommand: Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly SettingNameFeature _name; readonly StoragePathFeature _storagePath; - public ShowCommand(SeqConnectionFactory connectionFactory) + public ShowCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - _name = Enable(); _connection = Enable(); _storagePath = Enable(); @@ -42,7 +38,7 @@ public ShowCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var setting = await connection.Settings.FindNamedAsync(_name.Name); diff --git a/src/SeqCli/Cli/Commands/Shared/UpdateCommand.cs b/src/SeqCli/Cli/Commands/Shared/UpdateCommand.cs index fdf5bc44..de40eaa5 100644 --- a/src/SeqCli/Cli/Commands/Shared/UpdateCommand.cs +++ b/src/SeqCli/Cli/Commands/Shared/UpdateCommand.cs @@ -28,8 +28,6 @@ namespace SeqCli.Cli.Commands.Shared; abstract class UpdateCommand: Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly StoragePathFeature _storagePath; readonly string _resourceGroupName; @@ -38,9 +36,8 @@ abstract class UpdateCommand: Command string? _json; bool _jsonStdin; - protected UpdateCommand(SeqConnectionFactory connectionFactory, string commandGroupName, string resourceGroupName, string? entityName = null) + protected UpdateCommand(string commandGroupName, string resourceGroupName, string? entityName = null) { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); _resourceGroupName = resourceGroupName; _entityName = entityName ?? commandGroupName; @@ -61,7 +58,7 @@ protected UpdateCommand(SeqConnectionFactory connectionFactory, string commandGr protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); if (_json == null && !_jsonStdin) { diff --git a/src/SeqCli/Cli/Commands/Signal/CreateCommand.cs b/src/SeqCli/Cli/Commands/Signal/CreateCommand.cs index 539edfb2..16cb2f99 100644 --- a/src/SeqCli/Cli/Commands/Signal/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Signal/CreateCommand.cs @@ -29,8 +29,6 @@ namespace SeqCli.Cli.Commands.Signal; Example = "seqcli signal create -t 'Exceptions' -f \"@Exception is not null\"")] class CreateCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; @@ -40,10 +38,8 @@ class CreateCommand : Command string? _title, _description, _filter, _group; bool _isProtected, _noGrouping; - public CreateCommand(SeqConnectionFactory connectionFactory) + public CreateCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "t=|title=", "A title for the signal", @@ -87,7 +83,7 @@ public CreateCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var signal = await connection.Signals.TemplateAsync(); signal.OwnerId = null; diff --git a/src/SeqCli/Cli/Commands/Signal/ImportCommand.cs b/src/SeqCli/Cli/Commands/Signal/ImportCommand.cs index 66daf445..b9c79f78 100644 --- a/src/SeqCli/Cli/Commands/Signal/ImportCommand.cs +++ b/src/SeqCli/Cli/Commands/Signal/ImportCommand.cs @@ -28,7 +28,6 @@ namespace SeqCli.Cli.Commands.Signal; Example="seqcli signal import -i ./Exceptions.json")] class ImportCommand : Command { - readonly SeqConnectionFactory _connectionFactory; readonly FileInputFeature _fileInputFeature; readonly EntityOwnerFeature _entityOwner; readonly ConnectionFeature _connection; @@ -41,10 +40,8 @@ class ImportCommand : Command Converters = { new StringEnumConverter() } }); - public ImportCommand(SeqConnectionFactory connectionFactory) + public ImportCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "merge", "Update signals that have ids matching those in the imported data; the default is to always create new signals", @@ -59,7 +56,7 @@ public ImportCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); using var input = _fileInputFeature.OpenSingleInput(); var line = await input.ReadLineAsync(); diff --git a/src/SeqCli/Cli/Commands/Signal/ListCommand.cs b/src/SeqCli/Cli/Commands/Signal/ListCommand.cs index bfbdaa11..a035489c 100644 --- a/src/SeqCli/Cli/Commands/Signal/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Signal/ListCommand.cs @@ -24,18 +24,14 @@ namespace SeqCli.Cli.Commands.Signal; [Command("signal", "list", "List available signals", Example="seqcli signal list")] class ListCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly EntityIdentityFeature _entityIdentity; readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly EntityOwnerFeature _entityOwner; readonly StoragePathFeature _storagePath; - public ListCommand(SeqConnectionFactory connectionFactory) + public ListCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - _entityIdentity = Enable(new EntityIdentityFeature("signal", "list")); _entityOwner = Enable(new EntityOwnerFeature("signal", "list", "listed", _entityIdentity)); _output = Enable(); @@ -46,7 +42,7 @@ public ListCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var list = _entityIdentity.Id != null ? [await connection.Signals.FindAsync(_entityIdentity.Id)] : diff --git a/src/SeqCli/Cli/Commands/Signal/RemoveCommand.cs b/src/SeqCli/Cli/Commands/Signal/RemoveCommand.cs index 4727e91f..a989d7e5 100644 --- a/src/SeqCli/Cli/Commands/Signal/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/Signal/RemoveCommand.cs @@ -26,17 +26,13 @@ namespace SeqCli.Cli.Commands.Signal; Example = "seqcli signal remove -t 'Test Signal'")] class RemoveCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly EntityIdentityFeature _entityIdentity; readonly ConnectionFeature _connection; readonly EntityOwnerFeature _entityOwner; readonly StoragePathFeature _storagePath; - public RemoveCommand(SeqConnectionFactory connectionFactory) + public RemoveCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - _entityIdentity = Enable(new EntityIdentityFeature("signal", "remove")); _entityOwner = Enable(new EntityOwnerFeature("signal", "remove", "removed", _entityIdentity)); _connection = Enable(); @@ -52,7 +48,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var toRemove = _entityIdentity.Id != null ? [await connection.Signals.FindAsync(_entityIdentity.Id)] : diff --git a/src/SeqCli/Cli/Commands/Signal/UpdateCommand.cs b/src/SeqCli/Cli/Commands/Signal/UpdateCommand.cs index de734755..29f93fd4 100644 --- a/src/SeqCli/Cli/Commands/Signal/UpdateCommand.cs +++ b/src/SeqCli/Cli/Commands/Signal/UpdateCommand.cs @@ -20,6 +20,6 @@ namespace SeqCli.Cli.Commands.Signal; [Command("signal", "update", "Update an existing signal", Example="seqcli signal update --json '{...}'")] -class UpdateCommand(SeqConnectionFactory connectionFactory): - Shared.UpdateCommand(connectionFactory, "signal", nameof(SeqConnection.Signals)); +class UpdateCommand(): + Shared.UpdateCommand("signal", nameof(SeqConnection.Signals)); \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/TailCommand.cs b/src/SeqCli/Cli/Commands/TailCommand.cs index 1b7553c6..ae8c5907 100644 --- a/src/SeqCli/Cli/Commands/TailCommand.cs +++ b/src/SeqCli/Cli/Commands/TailCommand.cs @@ -26,17 +26,14 @@ namespace SeqCli.Cli.Commands; // ReSharper disable once UnusedType.Global class TailCommand : Command { - readonly SeqConnectionFactory _connectionFactory; readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly SignalExpressionFeature _signal; readonly StoragePathFeature _storagePath; string? _filter; - public TailCommand(SeqConnectionFactory connectionFactory) + public TailCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "f=|filter=", "An optional server-side filter to apply to the stream, for example `@Level = 'Error'`", @@ -54,7 +51,7 @@ protected override async Task Run() Console.CancelKeyPress += (_,_) => cancel.Cancel(); var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); string? strict = null; if (!string.IsNullOrWhiteSpace(_filter)) diff --git a/src/SeqCli/Cli/Commands/Template/ExportCommand.cs b/src/SeqCli/Cli/Commands/Template/ExportCommand.cs index e4bda0a0..71cefcf5 100644 --- a/src/SeqCli/Cli/Commands/Template/ExportCommand.cs +++ b/src/SeqCli/Cli/Commands/Template/ExportCommand.cs @@ -17,17 +17,14 @@ namespace SeqCli.Cli.Commands.Template; Example = "seqcli template export -o ./Templates")] class ExportCommand : Command { - readonly SeqConnectionFactory _connectionFactory; readonly ConnectionFeature _connection; readonly StoragePathFeature _storagePath; readonly HashSet _include = new(); string? _outputDir = "."; - public ExportCommand(SeqConnectionFactory connectionFactory) + public ExportCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "o=|output=", "The directory in which to write template files; the directory must exist; any existing files with " + @@ -59,7 +56,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var export = new TemplateSetExporter(connection, _include, _outputDir); await export.ExportTemplateSet(); diff --git a/src/SeqCli/Cli/Commands/Template/ImportCommand.cs b/src/SeqCli/Cli/Commands/Template/ImportCommand.cs index 88b2d607..2259b68d 100644 --- a/src/SeqCli/Cli/Commands/Template/ImportCommand.cs +++ b/src/SeqCli/Cli/Commands/Template/ImportCommand.cs @@ -23,7 +23,6 @@ namespace SeqCli.Cli.Commands.Template; Example = "seqcli template import -i ./Templates")] class ImportCommand : Command { - readonly SeqConnectionFactory _connectionFactory; readonly ConnectionFeature _connection; readonly PropertiesFeature _args; readonly StoragePathFeature _storagePath; @@ -32,9 +31,8 @@ class ImportCommand : Command string? _stateFile; bool _merge; - public ImportCommand(SeqConnectionFactory connectionFactory) + public ImportCommand() { - _connectionFactory = connectionFactory; Options.Add( "i=|input=", @@ -101,7 +99,7 @@ protected override async Task Run() })); var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var err = await TemplateSetImporter.ImportAsync(templates, connection, args, state, _merge); await TemplateImportState.SaveAsync(stateFile, state); diff --git a/src/SeqCli/Cli/Commands/User/CreateCommand.cs b/src/SeqCli/Cli/Commands/User/CreateCommand.cs index 21044540..09a8064c 100644 --- a/src/SeqCli/Cli/Commands/User/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/User/CreateCommand.cs @@ -28,8 +28,6 @@ namespace SeqCli.Cli.Commands.User; Example = "seqcli user create -n alice -d 'Alice Example' -r 'User (read/write)' --password-stdin")] class CreateCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; @@ -37,10 +35,8 @@ class CreateCommand : Command string? _username, _displayName, _roleTitle, _filter, _emailAddress, _password; bool _passwordStdin, _noPasswordChange; - public CreateCommand(SeqConnectionFactory connectionFactory) + public CreateCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "n=|name=", "A unique username for the user", @@ -89,7 +85,7 @@ public CreateCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var user = await connection.Users.TemplateAsync(); diff --git a/src/SeqCli/Cli/Commands/User/ListCommand.cs b/src/SeqCli/Cli/Commands/User/ListCommand.cs index 43417073..d7de035d 100644 --- a/src/SeqCli/Cli/Commands/User/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/User/ListCommand.cs @@ -24,17 +24,13 @@ namespace SeqCli.Cli.Commands.User; [Command("user", "list", "List users", Example="seqcli user list")] class ListCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly UserIdentityFeature _userIdentity; readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; - public ListCommand(SeqConnectionFactory connectionFactory) + public ListCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - _userIdentity = Enable(new UserIdentityFeature("list")); _output = Enable(); _storagePath = Enable(); @@ -44,7 +40,7 @@ public ListCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var list = _userIdentity.Id != null ? [await connection.Users.FindAsync(_userIdentity.Id)] : diff --git a/src/SeqCli/Cli/Commands/User/RemoveCommand.cs b/src/SeqCli/Cli/Commands/User/RemoveCommand.cs index ece8898a..082ec8e7 100644 --- a/src/SeqCli/Cli/Commands/User/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/User/RemoveCommand.cs @@ -26,16 +26,12 @@ namespace SeqCli.Cli.Commands.User; Example="seqcli user remove -n alice")] class RemoveCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly UserIdentityFeature _userIdentity; readonly ConnectionFeature _connection; readonly StoragePathFeature _storagePath; - public RemoveCommand(SeqConnectionFactory connectionFactory) + public RemoveCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - _userIdentity = Enable(new UserIdentityFeature("remove")); _connection = Enable(); _storagePath = Enable(); @@ -50,7 +46,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var toRemove = _userIdentity.Id != null ? [await connection.Users.FindAsync(_userIdentity.Id)] : diff --git a/src/SeqCli/Cli/Commands/User/UpdateCommand.cs b/src/SeqCli/Cli/Commands/User/UpdateCommand.cs index d440037c..de30ec3f 100644 --- a/src/SeqCli/Cli/Commands/User/UpdateCommand.cs +++ b/src/SeqCli/Cli/Commands/User/UpdateCommand.cs @@ -20,6 +20,6 @@ namespace SeqCli.Cli.Commands.User; [Command("user", "update", "Update an existing user", Example="seqcli user update --json '{...}'")] -class UpdateCommand(SeqConnectionFactory connectionFactory): - Shared.UpdateCommand(connectionFactory, "user", nameof(SeqConnection.Users)); +class UpdateCommand(): + Shared.UpdateCommand("user", nameof(SeqConnection.Users)); \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Workspace/CreateCommand.cs b/src/SeqCli/Cli/Commands/Workspace/CreateCommand.cs index 21c8457d..52103479 100644 --- a/src/SeqCli/Cli/Commands/Workspace/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Workspace/CreateCommand.cs @@ -15,8 +15,6 @@ namespace SeqCli.Cli.Commands.Workspace; Example = "seqcli workspace create -t 'My Workspace' -c signal-314159 -c dashboard-628318")] class CreateCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly StoragePathFeature _storagePath; @@ -25,10 +23,8 @@ class CreateCommand : Command bool _isProtected; readonly List _include = new(); - public CreateCommand(SeqConnectionFactory connectionFactory) + public CreateCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - Options.Add( "t=|title=", "A title for the workspace", @@ -57,7 +53,7 @@ public CreateCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var workspace = await connection.Workspaces.TemplateAsync(); workspace.OwnerId = null; diff --git a/src/SeqCli/Cli/Commands/Workspace/ListCommand.cs b/src/SeqCli/Cli/Commands/Workspace/ListCommand.cs index 84c2f777..9212b5e6 100644 --- a/src/SeqCli/Cli/Commands/Workspace/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Workspace/ListCommand.cs @@ -10,18 +10,14 @@ namespace SeqCli.Cli.Commands.Workspace; [Command("workspace", "list", "List available workspaces", Example = "seqcli workspace list")] class ListCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly EntityIdentityFeature _entityIdentity; readonly ConnectionFeature _connection; readonly OutputFormatFeature _output; readonly EntityOwnerFeature _entityOwner; readonly StoragePathFeature _storagePath; - public ListCommand(SeqConnectionFactory connectionFactory) + public ListCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - _entityIdentity = Enable(new EntityIdentityFeature("workspace", "list")); _entityOwner = Enable(new EntityOwnerFeature("workspace", "list", "listed", _entityIdentity)); _output = Enable(); @@ -32,7 +28,7 @@ public ListCommand(SeqConnectionFactory connectionFactory) protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var list = _entityIdentity.Id != null ? [await connection.Workspaces.FindAsync(_entityIdentity.Id)] : diff --git a/src/SeqCli/Cli/Commands/Workspace/RemoveCommand.cs b/src/SeqCli/Cli/Commands/Workspace/RemoveCommand.cs index 68d5b751..6208f068 100644 --- a/src/SeqCli/Cli/Commands/Workspace/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/Workspace/RemoveCommand.cs @@ -12,17 +12,13 @@ namespace SeqCli.Cli.Commands.Workspace; Example = "seqcli workspace remove -t 'My Workspace'")] class RemoveCommand : Command { - readonly SeqConnectionFactory _connectionFactory; - readonly EntityIdentityFeature _entityIdentity; readonly ConnectionFeature _connection; readonly EntityOwnerFeature _entityOwner; readonly StoragePathFeature _storagePath; - public RemoveCommand(SeqConnectionFactory connectionFactory) + public RemoveCommand() { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - _entityIdentity = Enable(new EntityIdentityFeature("workspace", "remove")); _entityOwner = Enable(new EntityOwnerFeature("workspace", "remove", "removed", _entityIdentity)); _connection = Enable(); @@ -38,7 +34,7 @@ protected override async Task Run() } var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = _connectionFactory.Connect(_connection, config); + var connection = SeqConnectionFactory.Connect(_connection, config); var toRemove = _entityIdentity.Id != null ? [await connection.Workspaces.FindAsync(_entityIdentity.Id)] : diff --git a/src/SeqCli/Cli/Commands/Workspace/UpdateCommand.cs b/src/SeqCli/Cli/Commands/Workspace/UpdateCommand.cs index c457506d..121b7511 100644 --- a/src/SeqCli/Cli/Commands/Workspace/UpdateCommand.cs +++ b/src/SeqCli/Cli/Commands/Workspace/UpdateCommand.cs @@ -20,6 +20,6 @@ namespace SeqCli.Cli.Commands.Workspace; [Command("workspace", "update", "Update an existing workspace", Example="seqcli workspace update --json '{...}'")] -class UpdateCommand(SeqConnectionFactory connectionFactory): - Shared.UpdateCommand(connectionFactory, "workspace", nameof(SeqConnection.Workspaces)); +class UpdateCommand(): + Shared.UpdateCommand("workspace", nameof(SeqConnection.Workspaces)); \ No newline at end of file diff --git a/src/SeqCli/Connection/SeqConnectionFactory.cs b/src/SeqCli/Connection/SeqConnectionFactory.cs index 6fdc8322..89596982 100644 --- a/src/SeqCli/Connection/SeqConnectionFactory.cs +++ b/src/SeqCli/Connection/SeqConnectionFactory.cs @@ -20,15 +20,15 @@ namespace SeqCli.Connection; -class SeqConnectionFactory +static class SeqConnectionFactory { - public SeqConnection Connect(ConnectionFeature connection, SeqCliConfig config) + public static SeqConnection Connect(ConnectionFeature connection, SeqCliConfig config) { var (url, apiKey) = GetConnectionDetails(connection, config); return new SeqConnection(url, apiKey); } - public (string? serverUrl, string? apiKey) GetConnectionDetails(ConnectionFeature connection, SeqCliConfig config) + public static (string? serverUrl, string? apiKey) GetConnectionDetails(ConnectionFeature connection, SeqCliConfig config) { if (connection == null) throw new ArgumentNullException(nameof(connection)); diff --git a/src/SeqCli/SeqCliModule.cs b/src/SeqCli/SeqCliModule.cs index 7fc0b8f2..93a2b71a 100644 --- a/src/SeqCli/SeqCliModule.cs +++ b/src/SeqCli/SeqCliModule.cs @@ -27,6 +27,5 @@ protected override void Load(ContainerBuilder builder) builder.RegisterAssemblyTypes(typeof(Program).GetTypeInfo().Assembly) .As() .WithMetadataFrom(); - builder.RegisterType(); } } From 70af0d6661c8914b891736d747026f6d202ec6be Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 18 Jul 2025 15:58:28 +1000 Subject: [PATCH 44/51] Use --storage to infer internal log path --- .../Cli/Commands/Forwarder/InstallCommand.cs | 4 +- .../Cli/Commands/Forwarder/RunCommand.cs | 5 +- src/SeqCli/Cli/Features/OutputFormat.cs | 168 ++++++++++++++++++ .../Cli/Features/OutputFormatFeature.cs | 154 +--------------- src/SeqCli/Cli/Features/StoragePathFeature.cs | 2 + .../Forwarder/ForwarderDiagnosticConfig.cs | 32 ---- ...iConfig.cs => SeqCliForwarderApiConfig.cs} | 2 +- .../Config/Forwarder/SeqCliForwarderConfig.cs | 6 +- .../SeqCliForwarderDiagnosticConfig.cs | 14 ++ ...fig.cs => SeqCliForwarderStorageConfig.cs} | 2 +- 10 files changed, 194 insertions(+), 195 deletions(-) create mode 100644 src/SeqCli/Cli/Features/OutputFormat.cs delete mode 100644 src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs rename src/SeqCli/Config/Forwarder/{ForwarderApiConfig.cs => SeqCliForwarderApiConfig.cs} (84%) create mode 100644 src/SeqCli/Config/Forwarder/SeqCliForwarderDiagnosticConfig.cs rename src/SeqCli/Config/Forwarder/{ForwarderStorageConfig.cs => SeqCliForwarderStorageConfig.cs} (79%) diff --git a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs index 330057de..03d6d32e 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs @@ -198,8 +198,8 @@ void Install() Console.WriteLine($"Granting {ServiceUsername} rights to {_storagePath.StorageRootPath}..."); GiveFullControl(_storagePath.StorageRootPath); - Console.WriteLine($"Granting {ServiceUsername} rights to {config.Forwarder.Diagnostics.InternalLogPath}..."); - GiveFullControl(config.Forwarder.Diagnostics.InternalLogPath); + Console.WriteLine($"Granting {ServiceUsername} rights to {_storagePath.InternalLogPath}..."); + GiveFullControl(_storagePath.InternalLogPath); var listenUri = MakeListenUriReservationPattern(config.Forwarder.Api.ListenUri); Console.WriteLine($"Adding URL reservation at {listenUri} for {ServiceUsername}..."); diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index f1bfc7f5..d0ab573f 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -25,7 +25,6 @@ using Microsoft.Extensions.Hosting; using SeqCli.Cli.Features; using SeqCli.Config; -using SeqCli.Config.Forwarder; using SeqCli.Connection; using SeqCli.Forwarder; using SeqCli.Forwarder.Util; @@ -85,7 +84,7 @@ protected override async Task Run(string[] unrecognized) { await using var logger = CreateLogger( LogEventLevel.Information, - ForwarderDiagnosticConfig.GetDefaultInternalLogPath()); + _storagePath.InternalLogPath); logger.Fatal(ex, "Failed to load configuration from {ConfigFilePath}", _storagePath.ConfigFilePath); return 1; @@ -100,7 +99,7 @@ protected override async Task Run(string[] unrecognized) Log.Logger = CreateLogger( config.Forwarder.Diagnostics.InternalLoggingLevel, - config.Forwarder.Diagnostics.InternalLogPath, + _storagePath.InternalLogPath, config.Forwarder.Diagnostics.InternalLogServerUri, config.Forwarder.Diagnostics.InternalLogServerApiKey); diff --git a/src/SeqCli/Cli/Features/OutputFormat.cs b/src/SeqCli/Cli/Features/OutputFormat.cs new file mode 100644 index 00000000..ce1f8894 --- /dev/null +++ b/src/SeqCli/Cli/Features/OutputFormat.cs @@ -0,0 +1,168 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Linq; +using Seq.Api.Model; +using SeqCli.Csv; +using SeqCli.Output; +using SeqCli.Util; +using Serilog; +using Serilog.Core; +using Serilog.Events; +using Serilog.Sinks.SystemConsole.Themes; +using Serilog.Templates.Themes; + +namespace SeqCli.Cli.Features; + +sealed class OutputFormat(bool json, bool noColor, bool forceColor) +{ + public const string DefaultOutputTemplate = + "[{Timestamp:o} {Level:u3}] {Message:lj} {Properties:j}{NewLine}{Exception}"; + + public static readonly ConsoleTheme DefaultAnsiTheme = AnsiConsoleTheme.Code; + + public static readonly ConsoleTheme DefaultTheme = + OperatingSystem.IsWindows() ? SystemConsoleTheme.Literate : DefaultAnsiTheme; + + static readonly TemplateTheme DefaultTemplateTheme = Serilog.Templates.Themes.TemplateTheme.Code; + + public bool Json => json; + + bool ApplyThemeToRedirectedOutput => noColor == false && forceColor; + + ConsoleTheme Theme + => noColor ? ConsoleTheme.None + : ApplyThemeToRedirectedOutput ? DefaultAnsiTheme + : DefaultTheme; + + TemplateTheme? TemplateTheme + => noColor ? null + : ApplyThemeToRedirectedOutput ? DefaultTemplateTheme + : null; + + public Logger CreateOutputLogger() + { + var outputConfiguration = new LoggerConfiguration() + .MinimumLevel.Is(LevelAlias.Minimum) + .Enrich.With(); + + if (json) + { + outputConfiguration.WriteTo.Console(OutputFormatter.Json(TemplateTheme)); + } + else + { + outputConfiguration.WriteTo.Console( + outputTemplate: DefaultOutputTemplate, + theme: Theme, + applyThemeToRedirectedOutput: ApplyThemeToRedirectedOutput); + } + + return outputConfiguration.CreateLogger(); + } + + public void WriteCsv(string csv) + { + if (noColor ) + { + Console.Write(csv); + } + else + { + var tokens = new CsvTokenizer().Tokenize(csv); + CsvWriter.WriteCsv(tokens, Theme, Console.Out, true); + } + } + + public void WriteEntity(Entity entity) + { + if (entity == null) throw new ArgumentNullException(nameof(entity)); + + var jo = JObject.FromObject( + entity, + JsonSerializer.CreateDefault(new JsonSerializerSettings { + DateParseHandling = DateParseHandling.None, + Converters = { + new StringEnumConverter() + } + })); + + if (json) + { + jo.Remove("Links"); + // Proof-of-concept; this is a very inefficient + // way to write colorized JSON ;) + + var writer = new LoggerConfiguration() + .Destructure.With() + .Enrich.With() + .WriteTo.Console( + outputTemplate: "{@Message:j}{NewLine}", + theme: Theme, + applyThemeToRedirectedOutput: ApplyThemeToRedirectedOutput) + .CreateLogger(); + writer.Information("{@Entity}", jo); + } + else + { + var dyn = (dynamic) jo; + Console.WriteLine($"{entity.Id} {dyn.Title ?? dyn.Name ?? dyn.Username ?? dyn.Expression}"); + } + } + + public void WriteObject(object value) + { + if (value == null) throw new ArgumentNullException(nameof(value)); + + if (json) + { + var jo = JObject.FromObject( + value, + JsonSerializer.CreateDefault(new JsonSerializerSettings { + DateParseHandling = DateParseHandling.None, + Converters = { + new StringEnumConverter() + } + })); + + // Using the same method of JSON colorization as above + + var writer = new LoggerConfiguration() + .Destructure.With() + .Enrich.With() + .WriteTo.Console( + outputTemplate: "{@Message:j}{NewLine}", + theme: Theme, + applyThemeToRedirectedOutput: ApplyThemeToRedirectedOutput) + .CreateLogger(); + writer.Information("{@Entity}", jo); + } + else + { + Console.WriteLine(value.ToString()); + } + } + + public void ListEntities(IEnumerable list) + { + foreach (var entity in list) + { + WriteEntity(entity); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/Cli/Features/OutputFormatFeature.cs b/src/SeqCli/Cli/Features/OutputFormatFeature.cs index a1c7fa5a..3d8da910 100644 --- a/src/SeqCli/Cli/Features/OutputFormatFeature.cs +++ b/src/SeqCli/Cli/Features/OutputFormatFeature.cs @@ -1,4 +1,4 @@ -// Copyright 2018 Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,21 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; -using System.Collections.Generic; -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; -using Newtonsoft.Json.Linq; -using Seq.Api.Model; using SeqCli.Config; -using SeqCli.Csv; -using SeqCli.Output; -using SeqCli.Util; -using Serilog; -using Serilog.Core; -using Serilog.Events; -using Serilog.Sinks.SystemConsole.Themes; -using Serilog.Templates.Themes; namespace SeqCli.Cli.Features; @@ -53,142 +39,4 @@ public override void Enable(OptionSet options) "Force redirected output to have ANSI color (unless `--no-color` is also specified)", _ => _forceColor = true); } -} - -sealed class OutputFormat(bool _json, bool _noColor, bool _forceColor) -{ - public const string DefaultOutputTemplate = - "[{Timestamp:o} {Level:u3}] {Message:lj} {Properties:j}{NewLine}{Exception}"; - - public static readonly ConsoleTheme DefaultAnsiTheme = AnsiConsoleTheme.Code; - - public static readonly ConsoleTheme DefaultTheme = - OperatingSystem.IsWindows() ? SystemConsoleTheme.Literate : DefaultAnsiTheme; - - static readonly TemplateTheme DefaultTemplateTheme = Serilog.Templates.Themes.TemplateTheme.Code; - - public bool Json => _json; - - bool ApplyThemeToRedirectedOutput => _noColor == false && _forceColor; - - ConsoleTheme Theme - => _noColor ? ConsoleTheme.None - : ApplyThemeToRedirectedOutput ? DefaultAnsiTheme - : DefaultTheme; - - TemplateTheme? TemplateTheme - => _noColor ? null - : ApplyThemeToRedirectedOutput ? DefaultTemplateTheme - : null; - - public Logger CreateOutputLogger() - { - var outputConfiguration = new LoggerConfiguration() - .MinimumLevel.Is(LevelAlias.Minimum) - .Enrich.With(); - - if (_json) - { - outputConfiguration.WriteTo.Console(OutputFormatter.Json(TemplateTheme)); - } - else - { - outputConfiguration.WriteTo.Console( - outputTemplate: DefaultOutputTemplate, - theme: Theme, - applyThemeToRedirectedOutput: ApplyThemeToRedirectedOutput); - } - - return outputConfiguration.CreateLogger(); - } - - public void WriteCsv(string csv) - { - if (_noColor ) - { - Console.Write(csv); - } - else - { - var tokens = new CsvTokenizer().Tokenize(csv); - CsvWriter.WriteCsv(tokens, Theme, Console.Out, true); - } - } - - public void WriteEntity(Entity entity) - { - if (entity == null) throw new ArgumentNullException(nameof(entity)); - - var jo = JObject.FromObject( - entity, - JsonSerializer.CreateDefault(new JsonSerializerSettings { - DateParseHandling = DateParseHandling.None, - Converters = { - new StringEnumConverter() - } - })); - - if (_json) - { - jo.Remove("Links"); - // Proof-of-concept; this is a very inefficient - // way to write colorized JSON ;) - - var writer = new LoggerConfiguration() - .Destructure.With() - .Enrich.With() - .WriteTo.Console( - outputTemplate: "{@Message:j}{NewLine}", - theme: Theme, - applyThemeToRedirectedOutput: ApplyThemeToRedirectedOutput) - .CreateLogger(); - writer.Information("{@Entity}", jo); - } - else - { - var dyn = (dynamic) jo; - Console.WriteLine($"{entity.Id} {dyn.Title ?? dyn.Name ?? dyn.Username ?? dyn.Expression}"); - } - } - - public void WriteObject(object value) - { - if (value == null) throw new ArgumentNullException(nameof(value)); - - if (_json) - { - var jo = JObject.FromObject( - value, - JsonSerializer.CreateDefault(new JsonSerializerSettings { - DateParseHandling = DateParseHandling.None, - Converters = { - new StringEnumConverter() - } - })); - - // Using the same method of JSON colorization as above - - var writer = new LoggerConfiguration() - .Destructure.With() - .Enrich.With() - .WriteTo.Console( - outputTemplate: "{@Message:j}{NewLine}", - theme: Theme, - applyThemeToRedirectedOutput: ApplyThemeToRedirectedOutput) - .CreateLogger(); - writer.Information("{@Entity}", jo); - } - else - { - Console.WriteLine(value.ToString()); - } - } - - public void ListEntities(IEnumerable list) - { - foreach (var entity in list) - { - WriteEntity(entity); - } - } } \ No newline at end of file diff --git a/src/SeqCli/Cli/Features/StoragePathFeature.cs b/src/SeqCli/Cli/Features/StoragePathFeature.cs index f2f6c2bc..e79f366a 100644 --- a/src/SeqCli/Cli/Features/StoragePathFeature.cs +++ b/src/SeqCli/Cli/Features/StoragePathFeature.cs @@ -26,6 +26,8 @@ public string StorageRootPath public string BufferPath => Path.Combine(StorageRootPath, "SeqCli", "Buffer"); + public string InternalLogPath => Path.Combine(StorageRootPath, "SeqCli", "Logs"); + public override void Enable(OptionSet options) { options.Add("storage=", diff --git a/src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs b/src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs deleted file mode 100644 index 6a9fab48..00000000 --- a/src/SeqCli/Config/Forwarder/ForwarderDiagnosticConfig.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System; -using System.IO; -using Serilog.Events; -// ReSharper disable UnusedAutoPropertyAccessor.Global -// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global - -namespace SeqCli.Config.Forwarder; - -public class ForwarderDiagnosticConfig -{ - public string InternalLogPath { get; set; } = GetDefaultInternalLogPath(); - public LogEventLevel InternalLoggingLevel { get; set; } = LogEventLevel.Information; - public string? InternalLogServerUri { get; set; } - public string? InternalLogServerApiKey { get; set; } - public bool ExposeIngestionLog { get; set; } - public bool IngestionLogShowDetail { get; set; } - - public static string GetDefaultInternalLogPath() - { - return Path.Combine( -#if WINDOWS - // Common, here, because the service may run as Local Service, which has no obvious home - // directory. - Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), -#else - // Specific to and writable by the current user. - Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), -#endif - "Seq", - "Logs"); - } -} \ No newline at end of file diff --git a/src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs b/src/SeqCli/Config/Forwarder/SeqCliForwarderApiConfig.cs similarity index 84% rename from src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs rename to src/SeqCli/Config/Forwarder/SeqCliForwarderApiConfig.cs index c0c27c19..136c6df1 100644 --- a/src/SeqCli/Config/Forwarder/ForwarderApiConfig.cs +++ b/src/SeqCli/Config/Forwarder/SeqCliForwarderApiConfig.cs @@ -2,7 +2,7 @@ namespace SeqCli.Config.Forwarder; // ReSharper disable AutoPropertyCanBeMadeGetOnly.Global -class ForwarderApiConfig +class SeqCliForwarderApiConfig { public string ListenUri { get; set; } = "http://127.0.0.1:15341"; } \ No newline at end of file diff --git a/src/SeqCli/Config/Forwarder/SeqCliForwarderConfig.cs b/src/SeqCli/Config/Forwarder/SeqCliForwarderConfig.cs index 065a4ae1..1043d0c8 100644 --- a/src/SeqCli/Config/Forwarder/SeqCliForwarderConfig.cs +++ b/src/SeqCli/Config/Forwarder/SeqCliForwarderConfig.cs @@ -4,7 +4,7 @@ namespace SeqCli.Config.Forwarder; class SeqCliForwarderConfig { - public ForwarderStorageConfig Storage { get; set; } = new(); - public ForwarderDiagnosticConfig Diagnostics { get; set; } = new(); - public ForwarderApiConfig Api { get; set; } = new(); + public SeqCliForwarderStorageConfig Storage { get; set; } = new(); + public SeqCliForwarderDiagnosticConfig Diagnostics { get; set; } = new(); + public SeqCliForwarderApiConfig Api { get; set; } = new(); } \ No newline at end of file diff --git a/src/SeqCli/Config/Forwarder/SeqCliForwarderDiagnosticConfig.cs b/src/SeqCli/Config/Forwarder/SeqCliForwarderDiagnosticConfig.cs new file mode 100644 index 00000000..326257d9 --- /dev/null +++ b/src/SeqCli/Config/Forwarder/SeqCliForwarderDiagnosticConfig.cs @@ -0,0 +1,14 @@ +using Serilog.Events; +// ReSharper disable UnusedAutoPropertyAccessor.Global +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global + +namespace SeqCli.Config.Forwarder; + +public class SeqCliForwarderDiagnosticConfig +{ + public LogEventLevel InternalLoggingLevel { get; set; } = LogEventLevel.Information; + public string? InternalLogServerUri { get; set; } + public string? InternalLogServerApiKey { get; set; } + public bool ExposeIngestionLog { get; set; } + public bool IngestionLogShowDetail { get; set; } +} \ No newline at end of file diff --git a/src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs b/src/SeqCli/Config/Forwarder/SeqCliForwarderStorageConfig.cs similarity index 79% rename from src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs rename to src/SeqCli/Config/Forwarder/SeqCliForwarderStorageConfig.cs index 5bc58044..8f88badf 100644 --- a/src/SeqCli/Config/Forwarder/ForwarderStorageConfig.cs +++ b/src/SeqCli/Config/Forwarder/SeqCliForwarderStorageConfig.cs @@ -2,7 +2,7 @@ namespace SeqCli.Config.Forwarder; // ReSharper disable AutoPropertyCanBeMadeGetOnly.Global -public class ForwarderStorageConfig +public class SeqCliForwarderStorageConfig { public ulong BufferSizeBytes { get; set; } = 67_108_864; } \ No newline at end of file From 59a608ec662a373b9ee115887c87d8e39ba05b04 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 18 Jul 2025 16:06:45 +1000 Subject: [PATCH 45/51] Opt out of authentication when end-to-end testing --- test/SeqCli.EndToEnd/Support/TestConfiguration.cs | 2 +- test/SeqCli.EndToEnd/TestDriverModule.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/SeqCli.EndToEnd/Support/TestConfiguration.cs b/test/SeqCli.EndToEnd/Support/TestConfiguration.cs index 1f33b97b..8841a006 100644 --- a/test/SeqCli.EndToEnd/Support/TestConfiguration.cs +++ b/test/SeqCli.EndToEnd/Support/TestConfiguration.cs @@ -41,7 +41,7 @@ public CaptiveProcess SpawnServerProcess(string storagePath) { var containerName = Guid.NewGuid().ToString("n"); const string containerRuntime = "docker"; - return new CaptiveProcess(containerRuntime, $"run --name {containerName} -d -e ACCEPT_EULA=Y -p {_serverListenPort}:80 datalust/seq:{imageTag}", stopCommandFullExePath: containerRuntime, stopCommandArgs: $"rm -f {containerName}"); + return new CaptiveProcess(containerRuntime, $"run --name {containerName} -d -e ACCEPT_EULA=Y SEQ_FIRSTRUN_NOAUTHENTICATION -p {_serverListenPort}:80 datalust/seq:{imageTag}", stopCommandFullExePath: containerRuntime, stopCommandArgs: $"rm -f {containerName}"); } return new CaptiveProcess("seq", commandWithArgs); diff --git a/test/SeqCli.EndToEnd/TestDriverModule.cs b/test/SeqCli.EndToEnd/TestDriverModule.cs index e0bbaf72..e52041a4 100644 --- a/test/SeqCli.EndToEnd/TestDriverModule.cs +++ b/test/SeqCli.EndToEnd/TestDriverModule.cs @@ -27,7 +27,7 @@ protected override void Load(ContainerBuilder builder) var testCases = _args.TestCases(); builder.RegisterAssemblyTypes(ThisAssembly) // ReSharper disable once AssignNullToNotNullAttribute - .Where(t => testCases == null || testCases.Length == 0 || testCases.Any(c => c.IsMatch(t.FullName))) + .Where(t => testCases.Length == 0 || testCases.Any(c => c.IsMatch(t.FullName))) .As() .WithMetadata(t => { From 346e524e09e6a8b7cd221a4468e1c47b8cb313c7 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 18 Jul 2025 16:07:07 +1000 Subject: [PATCH 46/51] Opt out properly :-) --- test/SeqCli.EndToEnd/Support/TestConfiguration.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/SeqCli.EndToEnd/Support/TestConfiguration.cs b/test/SeqCli.EndToEnd/Support/TestConfiguration.cs index 8841a006..a1780c71 100644 --- a/test/SeqCli.EndToEnd/Support/TestConfiguration.cs +++ b/test/SeqCli.EndToEnd/Support/TestConfiguration.cs @@ -41,7 +41,7 @@ public CaptiveProcess SpawnServerProcess(string storagePath) { var containerName = Guid.NewGuid().ToString("n"); const string containerRuntime = "docker"; - return new CaptiveProcess(containerRuntime, $"run --name {containerName} -d -e ACCEPT_EULA=Y SEQ_FIRSTRUN_NOAUTHENTICATION -p {_serverListenPort}:80 datalust/seq:{imageTag}", stopCommandFullExePath: containerRuntime, stopCommandArgs: $"rm -f {containerName}"); + return new CaptiveProcess(containerRuntime, $"run --name {containerName} -d -e ACCEPT_EULA=Y SEQ_FIRSTRUN_NOAUTHENTICATION=True -p {_serverListenPort}:80 datalust/seq:{imageTag}", stopCommandFullExePath: containerRuntime, stopCommandArgs: $"rm -f {containerName}"); } return new CaptiveProcess("seq", commandWithArgs); From 659501d30d212304817221763d265eef32c7b1e6 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 18 Jul 2025 16:50:09 +1000 Subject: [PATCH 47/51] Third time lucky --- src/SeqCli/Cli/CommandLineHost.cs | 5 +---- test/SeqCli.EndToEnd/Support/TestConfiguration.cs | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/SeqCli/Cli/CommandLineHost.cs b/src/SeqCli/Cli/CommandLineHost.cs index f221468b..27d46fc8 100644 --- a/src/SeqCli/Cli/CommandLineHost.cs +++ b/src/SeqCli/Cli/CommandLineHost.cs @@ -18,7 +18,6 @@ using System.Reflection; using System.Threading.Tasks; using Autofac.Features.Metadata; -using SeqCli.Cli.Features; using Serilog.Core; using Serilog.Events; @@ -26,12 +25,10 @@ namespace SeqCli.Cli; class CommandLineHost { - readonly StoragePathFeature _storagePathFeature; readonly List, CommandMetadata>> _availableCommands; - public CommandLineHost(IEnumerable, CommandMetadata>> availableCommands, StoragePathFeature storagePathFeature) + public CommandLineHost(IEnumerable, CommandMetadata>> availableCommands) { - _storagePathFeature = storagePathFeature; _availableCommands = availableCommands.ToList(); } diff --git a/test/SeqCli.EndToEnd/Support/TestConfiguration.cs b/test/SeqCli.EndToEnd/Support/TestConfiguration.cs index a1780c71..a95023d9 100644 --- a/test/SeqCli.EndToEnd/Support/TestConfiguration.cs +++ b/test/SeqCli.EndToEnd/Support/TestConfiguration.cs @@ -41,7 +41,7 @@ public CaptiveProcess SpawnServerProcess(string storagePath) { var containerName = Guid.NewGuid().ToString("n"); const string containerRuntime = "docker"; - return new CaptiveProcess(containerRuntime, $"run --name {containerName} -d -e ACCEPT_EULA=Y SEQ_FIRSTRUN_NOAUTHENTICATION=True -p {_serverListenPort}:80 datalust/seq:{imageTag}", stopCommandFullExePath: containerRuntime, stopCommandArgs: $"rm -f {containerName}"); + return new CaptiveProcess(containerRuntime, $"run --name {containerName} -d -e ACCEPT_EULA=Y -e SEQ_FIRSTRUN_NOAUTHENTICATION=True -p {_serverListenPort}:80 datalust/seq:{imageTag}", stopCommandFullExePath: containerRuntime, stopCommandArgs: $"rm -f {containerName}"); } return new CaptiveProcess("seq", commandWithArgs); From 79890149f5148b2eeb9e0f3fcf4d261d1185085f Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 18 Jul 2025 17:02:59 +1000 Subject: [PATCH 48/51] Fix help text --- src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs b/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs index 60c2d250..d82b15a8 100644 --- a/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs +++ b/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs @@ -29,7 +29,7 @@ class ServiceCredentialsFeature : CommandFeature public override void Enable(OptionSet options) { options.Add("u=|username=", - "The name of a Windows account to run the service under; if not specified the Local System account will be used", + "The name of a Windows account to run the service under; if not specified the `NT AUTHORITY\\LocalService` account will be used", v => Username = v.Trim()); options.Add("p=|password=", From 6cbbf034b2bc0866ed586bbe4f617817fbd07162 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 18 Jul 2025 17:04:59 +1000 Subject: [PATCH 49/51] Make tests build --- src/SeqCli/Config/SeqCliConnectionConfig.cs | 2 +- test/SeqCli.Tests/Cli/CommandLineHostTests.cs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/SeqCli/Config/SeqCliConnectionConfig.cs b/src/SeqCli/Config/SeqCliConnectionConfig.cs index c9a9f55d..0fd7bd6a 100644 --- a/src/SeqCli/Config/SeqCliConnectionConfig.cs +++ b/src/SeqCli/Config/SeqCliConnectionConfig.cs @@ -20,7 +20,7 @@ namespace SeqCli.Config; -public class SeqCliConnectionConfig +class SeqCliConnectionConfig { const string ProtectedDataPrefix = "pd."; diff --git a/test/SeqCli.Tests/Cli/CommandLineHostTests.cs b/test/SeqCli.Tests/Cli/CommandLineHostTests.cs index 377dff0c..b17ca6bb 100644 --- a/test/SeqCli.Tests/Cli/CommandLineHostTests.cs +++ b/test/SeqCli.Tests/Cli/CommandLineHostTests.cs @@ -27,7 +27,7 @@ public async Task CommandLineHostPicksCorrectCommand() new Lazy(() => new ActionCommand(() => executed.Add("test2"))), new CommandMetadata {Name = "test2", HelpText = "help"}) }; - var commandLineHost = new CommandLineHost(availableCommands, new StoragePathFeature()); + var commandLineHost = new CommandLineHost(availableCommands); await commandLineHost.Run(["test"],new LoggingLevelSwitch()); Assert.Equal("test", executed.Single()); @@ -43,7 +43,7 @@ public async Task PrereleaseCommandsAreIgnoredWithoutFlag() new Lazy(() => new ActionCommand(() => executed.Add("test"))), new CommandMetadata {Name = "test", HelpText = "help", IsPreview = true}), }; - var commandLineHost = new CommandLineHost(availableCommands, new StoragePathFeature()); + var commandLineHost = new CommandLineHost(availableCommands); var exit = await commandLineHost.Run(["test"],new LoggingLevelSwitch()); Assert.Equal(1, exit); Assert.Empty(executed); @@ -67,7 +67,7 @@ public async Task WhenMoreThanOneSubcommandAndTheUserRunsWithSubcommandEnsurePic new Lazy(() => new ActionCommand(() => executed.Add("test-subcommand2"))), new CommandMetadata {Name = "test", SubCommand = "subcommand2", HelpText = "help"}) }; - var commandLineHost = new CommandLineHost(availableCommands, new StoragePathFeature()); + var commandLineHost = new CommandLineHost(availableCommands); await commandLineHost.Run(["test", "subcommand2"], new LoggingLevelSwitch()); Assert.Equal("test-subcommand2", executed.First()); @@ -86,7 +86,7 @@ public async Task VerboseOptionSetsLoggingLevelToInformation() new CommandMetadata {Name = "test", HelpText = "help"}) }; - var commandLineHost = new CommandLineHost(availableCommands, new StoragePathFeature()); + var commandLineHost = new CommandLineHost(availableCommands); await commandLineHost.Run(["test", "--verbose"], levelSwitch); From 7df9ffff6383156d8a4a576d4f632893ed01b708 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 18 Jul 2025 17:09:26 +1000 Subject: [PATCH 50/51] Drop unused dependency --- src/SeqCli/Config/SeqCliEncryptionProviderConfig.cs | 2 +- src/SeqCli/Config/SeqCliOutputConfig.cs | 2 +- src/SeqCli/Encryptor/ExternalDataProtector.cs | 4 ++-- src/SeqCli/Encryptor/WindowsNativeDataProtector.cs | 2 +- src/SeqCli/Forwarder/Channel/ForwardingChannelEntry.cs | 2 +- src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs | 2 +- src/SeqCli/SeqCli.csproj | 1 - src/SeqCli/SeqCliModule.cs | 1 - 8 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/SeqCli/Config/SeqCliEncryptionProviderConfig.cs b/src/SeqCli/Config/SeqCliEncryptionProviderConfig.cs index d94750aa..e7273707 100644 --- a/src/SeqCli/Config/SeqCliEncryptionProviderConfig.cs +++ b/src/SeqCli/Config/SeqCliEncryptionProviderConfig.cs @@ -16,7 +16,7 @@ namespace SeqCli.Config; -public class SeqCliEncryptionProviderConfig +class SeqCliEncryptionProviderConfig { public string? Encryptor { get; set; } public string? EncryptorArgs { get; set; } diff --git a/src/SeqCli/Config/SeqCliOutputConfig.cs b/src/SeqCli/Config/SeqCliOutputConfig.cs index 652d7462..18529c90 100644 --- a/src/SeqCli/Config/SeqCliOutputConfig.cs +++ b/src/SeqCli/Config/SeqCliOutputConfig.cs @@ -17,7 +17,7 @@ namespace SeqCli.Config; -public class SeqCliOutputConfig +class SeqCliOutputConfig { public bool DisableColor { get; set; } public bool ForceColor { get; set; } diff --git a/src/SeqCli/Encryptor/ExternalDataProtector.cs b/src/SeqCli/Encryptor/ExternalDataProtector.cs index 3444b37b..0c84988b 100644 --- a/src/SeqCli/Encryptor/ExternalDataProtector.cs +++ b/src/SeqCli/Encryptor/ExternalDataProtector.cs @@ -7,7 +7,7 @@ namespace SeqCli.Encryptor; -public class ExternalDataProtector : IDataProtector +class ExternalDataProtector : IDataProtector { public ExternalDataProtector(SeqCliEncryptionProviderConfig providerConfig) { @@ -66,7 +66,7 @@ static int Invoke(string fullExePath, string? args, byte[] stdin, out byte[] std throw new InvalidOperationException("The process did not start."); var stderrBuf = new StringBuilder(); - process.ErrorDataReceived += (o, e) => + process.ErrorDataReceived += (_, e) => { if (e.Data == null) // ReSharper disable once AccessToDisposedClosure diff --git a/src/SeqCli/Encryptor/WindowsNativeDataProtector.cs b/src/SeqCli/Encryptor/WindowsNativeDataProtector.cs index 203d0f20..278431f3 100644 --- a/src/SeqCli/Encryptor/WindowsNativeDataProtector.cs +++ b/src/SeqCli/Encryptor/WindowsNativeDataProtector.cs @@ -5,7 +5,7 @@ namespace SeqCli.Encryptor; -public class WindowsNativeDataProtector : IDataProtector +class WindowsNativeDataProtector : IDataProtector { public byte[] Encrypt(byte[] unencrypted) { diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannelEntry.cs b/src/SeqCli/Forwarder/Channel/ForwardingChannelEntry.cs index 3f7d5fa9..6b038637 100644 --- a/src/SeqCli/Forwarder/Channel/ForwardingChannelEntry.cs +++ b/src/SeqCli/Forwarder/Channel/ForwardingChannelEntry.cs @@ -3,4 +3,4 @@ namespace SeqCli.Forwarder.Channel; -public readonly record struct ForwardingChannelEntry(ArraySegment Data, TaskCompletionSource CompletionSource); +readonly record struct ForwardingChannelEntry(ArraySegment Data, TaskCompletionSource CompletionSource); diff --git a/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs b/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs index 6375dcf9..1fd0f896 100644 --- a/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs +++ b/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs @@ -20,7 +20,7 @@ namespace SeqCli.Forwarder.Diagnostics; -public class InMemorySink : ILogEventSink +class InMemorySink : ILogEventSink { readonly int _queueLength; readonly ConcurrentQueue _queue = new(); diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index c4343a0f..627c4c85 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -57,7 +57,6 @@ - diff --git a/src/SeqCli/SeqCliModule.cs b/src/SeqCli/SeqCliModule.cs index 93a2b71a..2cfc7fb7 100644 --- a/src/SeqCli/SeqCliModule.cs +++ b/src/SeqCli/SeqCliModule.cs @@ -15,7 +15,6 @@ using System.Reflection; using Autofac; using SeqCli.Cli; -using SeqCli.Connection; namespace SeqCli; From bc82148bbbb9f9455ded4e633182668b90e76c88 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 18 Jul 2025 17:15:48 +1000 Subject: [PATCH 51/51] Fix print command --- src/SeqCli/Cli/Commands/PrintCommand.cs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/SeqCli/Cli/Commands/PrintCommand.cs b/src/SeqCli/Cli/Commands/PrintCommand.cs index 883685e0..51d845ab 100644 --- a/src/SeqCli/Cli/Commands/PrintCommand.cs +++ b/src/SeqCli/Cli/Commands/PrintCommand.cs @@ -34,16 +34,13 @@ class PrintCommand : Command { readonly FileInputFeature _fileInputFeature; readonly InvalidDataHandlingFeature _invalidDataHandlingFeature; + readonly StoragePathFeature _storage; string? _filter, _template = OutputFormat.DefaultOutputTemplate; - bool _noColor, _forceColor; + bool? _noColor, _forceColor; - public PrintCommand(SeqCliOutputConfig seqCliOutputConfig) + public PrintCommand() { - if (seqCliOutputConfig == null) throw new ArgumentNullException(nameof(seqCliOutputConfig)); - _noColor = seqCliOutputConfig.DisableColor; - _forceColor = seqCliOutputConfig.ForceColor; - _fileInputFeature = Enable(new FileInputFeature("CLEF file to read", allowMultiple: true)); Options.Add("f=|filter=", @@ -56,20 +53,24 @@ public PrintCommand(SeqCliOutputConfig seqCliOutputConfig) _invalidDataHandlingFeature = Enable(); + // These should be ported to use `OutputFormatFeature`. Options.Add("no-color", "Don't colorize text output", _ => _noColor = true); - Options.Add("force-color", "Force redirected output to have ANSI color (unless `--no-color` is also specified)", _ => _forceColor = true); + + _storage = Enable(); } protected override async Task Run() { + var config = RuntimeConfigurationLoader.Load(_storage); + var applyThemeToRedirectedOutput - = !_noColor && _forceColor; + = !(_noColor ?? config.Output.DisableColor) && (_forceColor ?? config.Output.ForceColor); var theme - = _noColor ? ConsoleTheme.None + = _noColor ?? config.Output.DisableColor ? ConsoleTheme.None : applyThemeToRedirectedOutput ? OutputFormat.DefaultAnsiTheme : OutputFormat.DefaultTheme;