From f828da64531eb425c700ad73f49675b07d6448ae Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 29 Feb 2024 14:31:17 +1000 Subject: [PATCH] Import existing Seq Forwarder codebase --- seqcli.sln | 1 - .../Cli/Commands/Forward/DumpCommand.cs | 13 - .../Cli/Commands/Forward/InstallCommand.cs | 17 -- .../Cli/Commands/Forward/RestartCommand.cs | 17 -- src/SeqCli/Cli/Commands/Forward/RunCommand.cs | 23 -- .../Cli/Commands/Forward/StartCommand.cs | 17 -- .../Cli/Commands/Forward/StatusCommand.cs | 17 -- .../Cli/Commands/Forward/StopCommand.cs | 17 -- .../Cli/Commands/Forward/TruncateCommand.cs | 13 - .../Cli/Commands/Forward/UninstallCommand.cs | 17 -- .../Cli/Commands/Forwarder/InstallCommand.cs | 262 ++++++++++++++++ .../Cli/Commands/Forwarder/RestartCommand.cs | 83 ++++++ .../Cli/Commands/Forwarder/RunCommand.cs | 258 ++++++++++++++++ .../Cli/Commands/Forwarder/StartCommand.cs | 67 +++++ .../Cli/Commands/Forwarder/StatusCommand.cs | 51 ++++ .../Cli/Commands/Forwarder/StopCommand.cs | 68 +++++ .../Cli/Commands/Forwarder/TruncateCommand.cs | 51 ++++ .../Commands/Forwarder/UninstallCommand.cs | 50 ++++ src/SeqCli/Cli/Features/ListenUriFeature.cs | 2 +- .../Cli/Features/ServiceCredentialsFeature.cs | 42 +++ src/SeqCli/Cli/Features/StoragePathFeature.cs | 6 +- .../Forwarder/Config/SeqForwarderApiConfig.cs | 21 ++ .../Forwarder/Config/SeqForwarderConfig.cs | 95 ++++++ .../Config/SeqForwarderDiagnosticConfig.cs | 44 +++ .../Config/SeqForwarderOutputConfig.cs | 56 ++++ .../Config/SeqForwarderStorageConfig.cs | 21 ++ .../DpapiMachineScopeDataProtection.cs | 46 +++ .../Cryptography/IStringDataProtector.cs | 8 + .../Cryptography/StringDataProtector.cs | 14 + .../Cryptography/UnprotectedStringData.cs | 22 ++ .../Forwarder/Diagnostics/InMemorySink.cs | 49 +++ .../Forwarder/Diagnostics/IngestionLog.cs | 65 ++++ .../Forwarder/Multiplexing/ActiveLogBuffer.cs | 38 +++ .../Multiplexing/ActiveLogBufferMap.cs | 235 +++++++++++++++ .../Multiplexing/HttpLogShipperFactory.cs | 41 +++ .../Multiplexing/ILogShipperFactory.cs | 24 ++ .../Multiplexing/InertLogShipperFactory.cs | 27 ++ .../Multiplexing/ServerResponseProxy.cs | 52 ++++ .../Forwarder/Properties/AssemblyInfo.cs | 3 + src/SeqCli/Forwarder/Schema/EventSchema.cs | 187 ++++++++++++ src/SeqCli/Forwarder/SeqForwarderModule.cs | 86 ++++++ .../SeqForwarderWindowsService.cs | 52 ++++ .../ExponentialBackoffConnectionSchedule.cs | 74 +++++ .../Forwarder/Shipper/HttpLogShipper.cs | 251 ++++++++++++++++ .../Forwarder/Shipper/InertLogShipper.cs | 31 ++ src/SeqCli/Forwarder/Shipper/LogShipper.cs | 25 ++ src/SeqCli/Forwarder/Shipper/SeqApi.cs | 21 ++ src/SeqCli/Forwarder/Storage/LogBuffer.cs | 280 ++++++++++++++++++ .../Forwarder/Storage/LogBufferEntry.cs | 24 ++ .../Forwarder/Util/AccountRightsHelper.cs | 193 ++++++++++++ src/SeqCli/Forwarder/Util/CaptiveProcess.cs | 82 +++++ .../Forwarder/Util/EnumerableExtensions.cs | 20 ++ .../Forwarder/Util/ExecutionEnvironment.cs | 20 ++ .../Forwarder/Util/ServiceConfiguration.cs | 111 +++++++ .../Forwarder/Util/UnclosableStreamWrapper.cs | 60 ++++ src/SeqCli/Forwarder/Util/WindowsProcess.cs | 51 ++++ .../Forwarder/Web/Api/ApiRootController.cs | 57 ++++ .../Forwarder/Web/Api/IngestionController.cs | 246 +++++++++++++++ .../Forwarder/Web/Host/ServerService.cs | 67 +++++ src/SeqCli/Forwarder/Web/Host/Startup.cs | 40 +++ .../Web/RequestProcessingException.cs | 30 ++ src/SeqCli/SeqCli.csproj | 23 +- .../Multiplexing/ActiveLogBufferMapTests.cs | 83 ++++++ .../Forwarder/Schema/EventSchemaTests.cs | 73 +++++ .../Shipper/ServerResponseProxyTests.cs | 49 +++ .../Forwarder/Storage/LogBufferTests.cs | 151 ++++++++++ test/SeqCli.Tests/SeqCli.Tests.csproj | 1 + test/SeqCli.Tests/Support/Some.cs | 17 ++ test/SeqCli.Tests/Support/TempFolder.cs | 51 ++++ 69 files changed, 4237 insertions(+), 172 deletions(-) delete mode 100644 src/SeqCli/Cli/Commands/Forward/DumpCommand.cs delete mode 100644 src/SeqCli/Cli/Commands/Forward/InstallCommand.cs delete mode 100644 src/SeqCli/Cli/Commands/Forward/RestartCommand.cs delete mode 100644 src/SeqCli/Cli/Commands/Forward/RunCommand.cs delete mode 100644 src/SeqCli/Cli/Commands/Forward/StartCommand.cs delete mode 100644 src/SeqCli/Cli/Commands/Forward/StatusCommand.cs delete mode 100644 src/SeqCli/Cli/Commands/Forward/StopCommand.cs delete mode 100644 src/SeqCli/Cli/Commands/Forward/TruncateCommand.cs delete mode 100644 src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs create mode 100644 src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs create mode 100644 src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs create mode 100644 src/SeqCli/Forwarder/Config/SeqForwarderApiConfig.cs create mode 100644 src/SeqCli/Forwarder/Config/SeqForwarderConfig.cs create mode 100644 src/SeqCli/Forwarder/Config/SeqForwarderDiagnosticConfig.cs create mode 100644 src/SeqCli/Forwarder/Config/SeqForwarderOutputConfig.cs create mode 100644 src/SeqCli/Forwarder/Config/SeqForwarderStorageConfig.cs create mode 100644 src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs create mode 100644 src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs create mode 100644 src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs create mode 100644 src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs create mode 100644 src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs create mode 100644 src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs create mode 100644 src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs create mode 100644 src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs create mode 100644 src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs create mode 100644 src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs create mode 100644 src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs create mode 100644 src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs create mode 100644 src/SeqCli/Forwarder/Properties/AssemblyInfo.cs create mode 100644 src/SeqCli/Forwarder/Schema/EventSchema.cs create mode 100644 src/SeqCli/Forwarder/SeqForwarderModule.cs create mode 100644 src/SeqCli/Forwarder/ServiceProcess/SeqForwarderWindowsService.cs create mode 100644 src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs create mode 100644 src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs create mode 100644 src/SeqCli/Forwarder/Shipper/InertLogShipper.cs create mode 100644 src/SeqCli/Forwarder/Shipper/LogShipper.cs create mode 100644 src/SeqCli/Forwarder/Shipper/SeqApi.cs create mode 100644 src/SeqCli/Forwarder/Storage/LogBuffer.cs create mode 100644 src/SeqCli/Forwarder/Storage/LogBufferEntry.cs create mode 100644 src/SeqCli/Forwarder/Util/AccountRightsHelper.cs create mode 100644 src/SeqCli/Forwarder/Util/CaptiveProcess.cs create mode 100644 src/SeqCli/Forwarder/Util/EnumerableExtensions.cs create mode 100644 src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs create mode 100644 src/SeqCli/Forwarder/Util/ServiceConfiguration.cs create mode 100644 src/SeqCli/Forwarder/Util/UnclosableStreamWrapper.cs create mode 100644 src/SeqCli/Forwarder/Util/WindowsProcess.cs create mode 100644 src/SeqCli/Forwarder/Web/Api/ApiRootController.cs create mode 100644 src/SeqCli/Forwarder/Web/Api/IngestionController.cs create mode 100644 src/SeqCli/Forwarder/Web/Host/ServerService.cs create mode 100644 src/SeqCli/Forwarder/Web/Host/Startup.cs create mode 100644 src/SeqCli/Forwarder/Web/RequestProcessingException.cs create mode 100644 test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs create mode 100644 test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs create mode 100644 test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs create mode 100644 test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs create mode 100644 test/SeqCli.Tests/Support/TempFolder.cs diff --git a/seqcli.sln b/seqcli.sln index 6c6104c5..bad9ba56 100644 --- a/seqcli.sln +++ b/seqcli.sln @@ -24,7 +24,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{3587B633-0 EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "asset", "asset", "{438A0DA5-F3CF-4FCE-B43A-B6DA2981D4AE}" ProjectSection(SolutionItems) = preProject - asset\SeqCliLicense.rtf = asset\SeqCliLicense.rtf asset\SeqCli.ico = asset\SeqCli.ico asset\SeqCli.png = asset\SeqCli.png EndProjectSection diff --git a/src/SeqCli/Cli/Commands/Forward/DumpCommand.cs b/src/SeqCli/Cli/Commands/Forward/DumpCommand.cs deleted file mode 100644 index 02b7c3cd..00000000 --- a/src/SeqCli/Cli/Commands/Forward/DumpCommand.cs +++ /dev/null @@ -1,13 +0,0 @@ -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -[Command("forward", "dump", "Print the complete log buffer contents as JSON", - Example = "seqcli forward dump")] -class DumpCommand : Command -{ - public DumpCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - } -} \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs deleted file mode 100644 index 4288dfab..00000000 --- a/src/SeqCli/Cli/Commands/Forward/InstallCommand.cs +++ /dev/null @@ -1,17 +0,0 @@ -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -#if WINDOWS - -[Command("forward", "install", "Install the Seq Forwarder as a Windows service", - Example = "seqcli forward install")] -class InstallCommand : Command -{ - public InstallCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - } -} - -#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs b/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs deleted file mode 100644 index 0d6cc8e7..00000000 --- a/src/SeqCli/Cli/Commands/Forward/RestartCommand.cs +++ /dev/null @@ -1,17 +0,0 @@ -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -#if WINDOWS - -[Command("forward", "restart", "Restart the Seq Forwarder Windows service", - Example = "seqcli forward restart")] -class RestartCommand : Command -{ - public RestartCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - } -} - -#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/RunCommand.cs b/src/SeqCli/Cli/Commands/Forward/RunCommand.cs deleted file mode 100644 index f2b0ed66..00000000 --- a/src/SeqCli/Cli/Commands/Forward/RunCommand.cs +++ /dev/null @@ -1,23 +0,0 @@ -using SeqCli.Cli.Features; -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -[Command("forward", "run", "Run the Seq Forwarder server interactively", - Example = "seqcli forward run")] -class RunCommand : Command -{ -#pragma warning disable CS0414 // Field is assigned but its value is never used - bool _noLogo; -#pragma warning restore CS0414 // Field is assigned but its value is never used - readonly StoragePathFeature _storagePath; - readonly ListenUriFeature _listenUri; - - public RunCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - Options.Add("nologo", _ => _noLogo = true); - _storagePath = Enable(); - _listenUri = Enable(); - } -} \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/StartCommand.cs b/src/SeqCli/Cli/Commands/Forward/StartCommand.cs deleted file mode 100644 index 6ddb52dc..00000000 --- a/src/SeqCli/Cli/Commands/Forward/StartCommand.cs +++ /dev/null @@ -1,17 +0,0 @@ -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -#if WINDOWS - -[Command("forward", "start", "Start the Seq Forwarder Windows service", - Example = "seqcli forward start")] -class StartCommand : Command -{ - public StartCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - } -} - -#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs b/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs deleted file mode 100644 index d9f1f736..00000000 --- a/src/SeqCli/Cli/Commands/Forward/StatusCommand.cs +++ /dev/null @@ -1,17 +0,0 @@ -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -#if WINDOWS - -[Command("forward", "status", "Show the status of the Seq Forwarder service", - Example = "seqcli forward status")] -class StatusCommand : Command -{ - public StatusCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - } -} - -#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/StopCommand.cs b/src/SeqCli/Cli/Commands/Forward/StopCommand.cs deleted file mode 100644 index 06bc93b4..00000000 --- a/src/SeqCli/Cli/Commands/Forward/StopCommand.cs +++ /dev/null @@ -1,17 +0,0 @@ -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -#if WINDOWS - -[Command("forward", "stop", "Stop the Seq Forwarder service", - Example = "seqcli forward stop")] -class StopCommand : Command -{ - public StopCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - } -} - -#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/TruncateCommand.cs b/src/SeqCli/Cli/Commands/Forward/TruncateCommand.cs deleted file mode 100644 index 5499424f..00000000 --- a/src/SeqCli/Cli/Commands/Forward/TruncateCommand.cs +++ /dev/null @@ -1,13 +0,0 @@ -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -[Command("forward", "truncate", "Clear the log buffer contents", - Example = "seqcli forward truncate")] -class TruncateCommand : Command -{ - public TruncateCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - } -} \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs deleted file mode 100644 index a8fa690f..00000000 --- a/src/SeqCli/Cli/Commands/Forward/UninstallCommand.cs +++ /dev/null @@ -1,17 +0,0 @@ -using SeqCli.Config; -using SeqCli.Connection; - -namespace SeqCli.Cli.Commands.Forward; - -#if WINDOWS - -[Command("forward", "uninstall", "Uninstall the Windows service", - Example = "seqcli forward uninstall")] -internal class UninstallCommand : Command -{ - public UninstallCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) - { - } -} - -#endif \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs new file mode 100644 index 00000000..7dbc9873 --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs @@ -0,0 +1,262 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System; +using System.IO; +using System.Runtime.InteropServices; +using System.Security.AccessControl; +using System.ServiceProcess; +using Seq.Forwarder.Cli.Features; +using Seq.Forwarder.Config; +using Seq.Forwarder.ServiceProcess; +using Seq.Forwarder.Util; + +// ReSharper disable once ClassNeverInstantiated.Global + +namespace Seq.Forwarder.Cli.Commands +{ + [Command("forwarder", "install", "Install the Seq Forwarder as a Windows service")] + class InstallCommand : Command + { + readonly StoragePathFeature _storagePath; + readonly ServiceCredentialsFeature _serviceCredentials; + readonly ListenUriFeature _listenUri; + + bool _setup; + + public InstallCommand() + { + _storagePath = Enable(); + _listenUri = Enable(); + _serviceCredentials = Enable(); + + Options.Add( + "setup", + "Install and start the service only if it does not exist; otherwise reconfigure the binary location", + v => _setup = true); + } + + string ServiceUsername => _serviceCredentials.IsUsernameSpecified ? _serviceCredentials.Username : "NT AUTHORITY\\LocalService"; + + protected override int Run(TextWriter cout) + { + try + { + if (!_setup) + { + Install(cout); + return 0; + } + + var exit = Setup(cout); + if (exit == 0) + { + Console.ForegroundColor = ConsoleColor.Green; + Console.WriteLine("Setup completed successfully."); + Console.ResetColor(); + } + return exit; + } + catch (DirectoryNotFoundException dex) + { + cout.WriteLine("Could not install the service, directory not found: " + dex.Message); + return -1; + } + catch (Exception ex) + { + cout.WriteLine("Could not install the service: " + ex.Message); + return -1; + } + } + + int Setup(TextWriter cout) + { + ServiceController controller; + try + { + cout.WriteLine("Checking the status of the Seq Forwarder service..."); + + controller = new ServiceController(SeqForwarderWindowsService.WindowsServiceName); + cout.WriteLine("Status is {0}", controller.Status); + } + catch (InvalidOperationException) + { + Install(cout); + var controller2 = new ServiceController(SeqForwarderWindowsService.WindowsServiceName); + return Start(controller2, cout); + } + + cout.WriteLine("Service is installed; checking path and dependency configuration..."); + Reconfigure(controller, cout); + + if (controller.Status != ServiceControllerStatus.Running) + return Start(controller, cout); + + return 0; + } + + static void Reconfigure(ServiceController controller, TextWriter cout) + { + var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); + if (0 != CaptiveProcess.Run(sc, "config \"" + controller.ServiceName + "\" depend= Winmgmt/Tcpip/CryptSvc", cout.WriteLine, cout.WriteLine)) + cout.WriteLine("Could not reconfigure service dependencies; ignoring."); + + if (!ServiceConfiguration.GetServiceBinaryPath(controller, cout, out var path)) + return; + + var current = "\"" + Path.Combine(AppDomain.CurrentDomain.BaseDirectory!, Program.BinaryName) + "\""; + if (path.StartsWith(current)) + return; + + var seqRun = path.IndexOf(Program.BinaryName + "\" run", StringComparison.OrdinalIgnoreCase); + if (seqRun == -1) + { + cout.WriteLine("Current binary path is an unrecognized format."); + return; + } + + cout.WriteLine("Existing service binary path is: {0}", path); + + var trimmed = path.Substring((seqRun + Program.BinaryName + " ").Length); + var newPath = current + trimmed; + cout.WriteLine("Updating service binary path configuration to: {0}", newPath); + + var escaped = newPath.Replace("\"", "\\\""); + if (0 != CaptiveProcess.Run(sc, "config \"" + controller.ServiceName + "\" binPath= \"" + escaped + "\"", cout.WriteLine, cout.WriteLine)) + { + cout.WriteLine("Could not reconfigure service path; ignoring."); + return; + } + + cout.WriteLine("Service binary path reconfigured successfully."); + } + + static int Start(ServiceController controller, TextWriter cout) + { + controller.Start(); + + if (controller.Status != ServiceControllerStatus.Running) + { + cout.WriteLine("Waiting up to 60 seconds for the service to start (currently: " + controller.Status + ")..."); + controller.WaitForStatus(ServiceControllerStatus.Running, TimeSpan.FromSeconds(60)); + } + + if (controller.Status == ServiceControllerStatus.Running) + { + cout.WriteLine("Started."); + return 0; + } + + cout.WriteLine("The service hasn't started successfully."); + return -1; + } + + [DllImport("shlwapi.dll")] + static extern bool PathIsNetworkPath(string pszPath); + + void Install(TextWriter cout) + { + cout.WriteLine("Installing service..."); + + if (PathIsNetworkPath(_storagePath.StorageRootPath)) + throw new ArgumentException("Seq requires a local (or SAN) storage location; network shares are not supported."); + + cout.WriteLine($"Updating the configuration in {_storagePath.ConfigFilePath}..."); + var config = SeqForwarderConfig.ReadOrInit(_storagePath.ConfigFilePath); + + if (!string.IsNullOrEmpty(_listenUri.ListenUri)) + { + config.Api.ListenUri = _listenUri.ListenUri; + SeqForwarderConfig.Write(_storagePath.ConfigFilePath, config); + } + + if (_serviceCredentials.IsUsernameSpecified) + { + if (!_serviceCredentials.IsPasswordSpecified) + throw new ArgumentException( + "If a service user account is specified, a password for the account must also be specified."); + + // https://technet.microsoft.com/en-us/library/cc794944(v=ws.10).aspx + cout.WriteLine($"Ensuring {_serviceCredentials.Username} is granted 'Log on as a Service' rights..."); + AccountRightsHelper.EnsureServiceLogOnRights(_serviceCredentials.Username); + } + + cout.WriteLine($"Granting {ServiceUsername} rights to {_storagePath.StorageRootPath}..."); + GiveFullControl(_storagePath.StorageRootPath); + + cout.WriteLine($"Granting {ServiceUsername} rights to {config.Diagnostics.InternalLogPath}..."); + GiveFullControl(config.Diagnostics.InternalLogPath); + + var listenUri = MakeListenUriReservationPattern(config.Api.ListenUri); + cout.WriteLine($"Adding URL reservation at {listenUri} for {ServiceUsername}..."); + var netshResult = CaptiveProcess.Run("netsh", $"http add urlacl url={listenUri} user=\"{ServiceUsername}\"", cout.WriteLine, cout.WriteLine); + if (netshResult != 0) + cout.WriteLine($"Could not add URL reservation for {listenUri}: `netsh` returned {netshResult}; ignoring"); + + var exePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory!, Program.BinaryName); + var forwarderRunCmdline = $"\"{exePath}\" run --storage=\"{_storagePath.StorageRootPath}\""; + + var binPath = forwarderRunCmdline.Replace("\"", "\\\""); + + var scCmdline = "create \"" + SeqForwarderWindowsService.WindowsServiceName + "\"" + + " binPath= \"" + binPath + "\"" + + " start= auto" + + " depend= Winmgmt/Tcpip/CryptSvc"; + + if (_serviceCredentials.IsUsernameSpecified) + scCmdline += $" obj= {_serviceCredentials.Username} password= {_serviceCredentials.Password}"; + + var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); + if (0 != CaptiveProcess.Run(sc, scCmdline, cout.WriteLine, cout.WriteLine)) + { + throw new ArgumentException("Service setup failed"); + } + + cout.WriteLine("Setting service restart policy..."); + if (0 != CaptiveProcess.Run(sc, $"failure \"{SeqForwarderWindowsService.WindowsServiceName}\" actions= restart/60000/restart/60000/restart/60000// reset= 600000", cout.WriteLine, cout.WriteLine)) + cout.WriteLine("Could not set service restart policy; ignoring"); + cout.WriteLine("Setting service description..."); + if (0 != CaptiveProcess.Run(sc, $"description \"{SeqForwarderWindowsService.WindowsServiceName}\" \"Durable storage and forwarding of application log events\"", cout.WriteLine, cout.WriteLine)) + cout.WriteLine("Could not set service description; ignoring"); + + cout.WriteLine("Service installed successfully."); + } + + void GiveFullControl(string target) + { + if (target == null) throw new ArgumentNullException(nameof(target)); + + if (!Directory.Exists(target)) + Directory.CreateDirectory(target); + + var storageInfo = new DirectoryInfo(target); + var storageAccessControl = storageInfo.GetAccessControl(); + storageAccessControl.AddAccessRule(new FileSystemAccessRule(ServiceUsername, + FileSystemRights.FullControl, AccessControlType.Allow)); + storageInfo.SetAccessControl(storageAccessControl); + } + + static string MakeListenUriReservationPattern(string uri) + { + var listenUri = uri.Replace("localhost", "+"); + if (!listenUri.EndsWith("/")) + listenUri += "/"; + return listenUri; + } + } +} + +#endif diff --git a/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs new file mode 100644 index 00000000..fb97172a --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs @@ -0,0 +1,83 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System; +using System.IO; +using System.ServiceProcess; +using Seq.Forwarder.ServiceProcess; + +// ReSharper disable UnusedType.Global + +namespace Seq.Forwarder.Cli.Commands +{ + [Command("forwarder", "restart", "Restart the Windows service")] + class RestartCommand : Command + { + protected override int Run(TextWriter cout) + { + try + { + var controller = new ServiceController(SeqForwarderWindowsService.WindowsServiceName); + + if (controller.Status != ServiceControllerStatus.Stopped) + { + cout.WriteLine("Stopping {0}...", controller.ServiceName); + controller.Stop(); + + if (controller.Status != ServiceControllerStatus.Stopped) + { + cout.WriteLine("Waiting up to 60 seconds for the service to stop (currently: " + + controller.Status + ")..."); + controller.WaitForStatus(ServiceControllerStatus.Stopped, TimeSpan.FromSeconds(60)); + } + + if (controller.Status != ServiceControllerStatus.Stopped) + { + cout.WriteLine("The service hasn't stopped successfully."); + return -1; + } + } + + cout.WriteLine("Starting {0}...", controller.ServiceName); + controller.Start(); + + if (controller.Status != ServiceControllerStatus.Running) + { + cout.WriteLine("Waiting up to 15 seconds for the service to start (currently: " + controller.Status + ")..."); + controller.WaitForStatus(ServiceControllerStatus.Running, TimeSpan.FromSeconds(15)); + } + + if (controller.Status == ServiceControllerStatus.Running) + { + cout.WriteLine("Started."); + return 0; + } + + cout.WriteLine("The service hasn't started successfully."); + return -1; + } + catch (Exception ex) + { + cout.WriteLine(ex.Message); + if (ex.InnerException != null) + cout.WriteLine(ex.InnerException.Message); + return 1; + } + } + } +} + +#endif diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs new file mode 100644 index 00000000..0af8e266 --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -0,0 +1,258 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Autofac; +using Seq.Forwarder.Config; +using Serilog; +using Serilog.Events; +using Serilog.Formatting.Compact; +using System; +using System.IO; +using System.Net; +using System.Threading.Tasks; +using Autofac.Extensions.DependencyInjection; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Hosting; +using Seq.Forwarder.Util; +using Seq.Forwarder.Web.Host; +using SeqCli.Cli; +using SeqCli.Cli.Features; +using Serilog.Core; + +// ReSharper disable UnusedType.Global + +namespace Seq.Forwarder.Cli.Commands +{ + [Command("forwarder", "run", "Run the server interactively")] + class RunCommand : Command + { + readonly StoragePathFeature _storagePath; + readonly ListenUriFeature _listenUri; + + bool _noLogo; + + public RunCommand() + { + Options.Add("nologo", _ => _noLogo = true); + _storagePath = Enable(); + _listenUri = Enable(); + } + + protected override async Task Run(string[] unrecognized) + { + if (Environment.UserInteractive) + { + if (!_noLogo) + { + WriteBanner(); + Console.WriteLine(); + } + + Console.WriteLine("Running as server; press Ctrl+C to exit."); + Console.WriteLine(); + } + + SeqForwarderConfig config; + + try + { + config = SeqForwarderConfig.ReadOrInit(_storagePath.ConfigFilePath); + } + catch (Exception ex) + { + await using var logger = CreateLogger( + LogEventLevel.Information, + SeqForwarderDiagnosticConfig.GetDefaultInternalLogPath()); + + logger.Fatal(ex, "Failed to load configuration from {ConfigFilePath}", _storagePath.ConfigFilePath); + return 1; + } + + Log.Logger = CreateLogger( + config.Diagnostics.InternalLoggingLevel, + config.Diagnostics.InternalLogPath, + config.Diagnostics.InternalLogServerUri, + config.Diagnostics.InternalLogServerApiKey); + + var listenUri = _listenUri.ListenUri ?? config.Api.ListenUri; + + try + { + ILifetimeScope? container = null; + using var host = new HostBuilder() + .UseSerilog() + .UseServiceProviderFactory(new AutofacServiceProviderFactory()) + .ConfigureContainer(builder => + { + builder.RegisterBuildCallback(ls => container = ls); + builder.RegisterModule(new SeqForwarderModule(_storagePath.BufferPath, config)); + }) + .ConfigureWebHostDefaults(web => + { + web.UseStartup(); + web.UseKestrel(options => + { + options.AddServerHeader = false; + options.AllowSynchronousIO = true; + }) + .ConfigureKestrel(options => + { + var apiListenUri = new Uri(listenUri); + + var ipAddress = apiListenUri.HostNameType switch + { + UriHostNameType.Basic => IPAddress.Any, + UriHostNameType.Dns => IPAddress.Any, + UriHostNameType.IPv4 => IPAddress.Parse(apiListenUri.Host), + UriHostNameType.IPv6 => IPAddress.Parse(apiListenUri.Host), + _ => throw new NotSupportedException($"Listen URI type `{apiListenUri.HostNameType}` is not supported.") + }; + + if (apiListenUri.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase)) + { + options.Listen(ipAddress, apiListenUri.Port, listenOptions => + { +#if WINDOWS + listenOptions.UseHttps(StoreName.My, apiListenUri.Host, + location: StoreLocation.LocalMachine, allowInvalid: true); +#else + listenOptions.UseHttps(); +#endif + }); + } + else + { + options.Listen(ipAddress, apiListenUri.Port); + } + }); + }) + .Build(); + + if (container == null) throw new Exception("Host did not build container."); + + var service = container.Resolve( + new TypedParameter(typeof(IHost), host), + new NamedParameter("listenUri", listenUri)); + + var exit = ExecutionEnvironment.SupportsStandardIO + ? RunStandardIO(service, Console.Out) + : RunService(service); + + return exit; + } + catch (Exception ex) + { + Log.Fatal(ex, "Unhandled exception"); + return -1; + } + finally + { + await Log.CloseAndFlushAsync(); + } + } + + static Logger CreateLogger( + LogEventLevel internalLoggingLevel, + string internalLogPath, + string? internalLogServerUri = null, + string? internalLogServerApiKey = null) + { + var loggerConfiguration = new LoggerConfiguration() + .Enrich.FromLogContext() + .Enrich.WithProperty("MachineName", Environment.MachineName) + .Enrich.WithProperty("Application", "Seq Forwarder") + .MinimumLevel.Is(internalLoggingLevel) + .MinimumLevel.Override("Microsoft", LogEventLevel.Warning) + .WriteTo.File( + new RenderedCompactJsonFormatter(), + GetRollingLogFilePathFormat(internalLogPath), + rollingInterval: RollingInterval.Day, + fileSizeLimitBytes: 1024 * 1024); + + if (Environment.UserInteractive) + loggerConfiguration.WriteTo.Console(restrictedToMinimumLevel: LogEventLevel.Information); + + if (!string.IsNullOrWhiteSpace(internalLogServerUri)) + loggerConfiguration.WriteTo.Seq( + internalLogServerUri, + apiKey: internalLogServerApiKey); + + return loggerConfiguration.CreateLogger(); + } + + static string GetRollingLogFilePathFormat(string internalLogPath) + { + if (internalLogPath == null) throw new ArgumentNullException(nameof(internalLogPath)); + + return Path.Combine(internalLogPath, "seq-forwarder-.log"); + } + + static int RunService(ServerService service) + { +#if WINDOWS + System.ServiceProcess.ServiceBase.Run([ + new ServiceProcess.SeqForwarderWindowsService(service) + ]); + return 0; +#else + throw new NotSupportedException("Windows services are not supported on this platform."); +#endif + } + + static int RunStandardIO(ServerService service, TextWriter cout) + { + service.Start(); + + try + { + Console.TreatControlCAsInput = true; + var k = Console.ReadKey(true); + while (k.Key != ConsoleKey.C || !k.Modifiers.HasFlag(ConsoleModifiers.Control)) + k = Console.ReadKey(true); + + cout.WriteLine("Ctrl+C pressed; stopping..."); + Console.TreatControlCAsInput = false; + } + catch (Exception ex) + { + Log.Debug(ex, "Console not attached, waiting for any input"); + Console.Read(); + } + + service.Stop(); + + return 0; + } + + static void WriteBanner() + { + Write("─", ConsoleColor.DarkGray, 47); + Console.WriteLine(); + Write(" Seq Forwarder", ConsoleColor.White); + Write(" ──", ConsoleColor.DarkGray); + Write(" © 2024 Datalust Pty Ltd", ConsoleColor.Gray); + Console.WriteLine(); + Write("─", ConsoleColor.DarkGray, 47); + Console.WriteLine(); + } + + static void Write(string s, ConsoleColor color, int repeats = 1) + { + Console.ForegroundColor = color; + for (var i = 0; i < repeats; ++i) + Console.Write(s); + Console.ResetColor(); + } + } +} diff --git a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs new file mode 100644 index 00000000..2b48ca4f --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs @@ -0,0 +1,67 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System; +using System.IO; +using System.ServiceProcess; +using Seq.Forwarder.ServiceProcess; + +namespace Seq.Forwarder.Cli.Commands +{ + [Command("forwarder", "start", "Start the Windows service")] + class StartCommand : Command + { + protected override int Run(TextWriter cout) + { + try + { + var controller = new ServiceController(SeqForwarderWindowsService.WindowsServiceName); + if (controller.Status != ServiceControllerStatus.Stopped) + { + cout.WriteLine("Cannot start {0}, current status is: {1}", controller.ServiceName, controller.Status); + return -1; + } + + cout.WriteLine("Starting {0}...", controller.ServiceName); + controller.Start(); + + if (controller.Status != ServiceControllerStatus.Running) + { + cout.WriteLine("Waiting up to 15 seconds for the service to start (currently: " + controller.Status + ")..."); + controller.WaitForStatus(ServiceControllerStatus.Running, TimeSpan.FromSeconds(15)); + } + + if (controller.Status == ServiceControllerStatus.Running) + { + cout.WriteLine("Started."); + return 0; + } + + cout.WriteLine("The service hasn't started successfully."); + return -1; + } + catch (Exception ex) + { + cout.WriteLine(ex.Message); + if (ex.InnerException != null) + cout.WriteLine(ex.InnerException.Message); + return -1; + } + } + } +} + +#endif diff --git a/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs new file mode 100644 index 00000000..c9261d93 --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs @@ -0,0 +1,51 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System; +using System.IO; +using System.ServiceProcess; +using Seq.Forwarder.ServiceProcess; + +namespace Seq.Forwarder.Cli.Commands +{ + [Command("forwarder", "status", "Show the status of the Seq Forwarder service")] + class StatusCommand : Command + { + protected override int Run(TextWriter cout) + { + try + { + var controller = new ServiceController(SeqForwarderWindowsService.WindowsServiceName); + cout.WriteLine("The Seq Forwarder service is installed and {0}.", controller.Status.ToString().ToLowerInvariant()); + } + catch (InvalidOperationException) + { + cout.WriteLine("The Seq Forwarder service is not installed."); + } + catch (Exception ex) + { + cout.WriteLine(ex.Message); + if (ex.InnerException != null) + cout.WriteLine(ex.InnerException.Message); + return 1; + } + + return 0; + } + } +} + +#endif diff --git a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs new file mode 100644 index 00000000..66c16637 --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs @@ -0,0 +1,68 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System; +using System.IO; +using System.ServiceProcess; +using Seq.Forwarder.ServiceProcess; + +namespace Seq.Forwarder.Cli.Commands +{ + [Command("forwarder", "stop", "Stop the Windows service")] + class StopCommand : Command + { + protected override int Run(TextWriter cout) + { + try + { + var controller = new ServiceController(SeqForwarderWindowsService.WindowsServiceName); + + if (controller.Status != ServiceControllerStatus.Running) + { + cout.WriteLine("Cannot stop {0}, current status is: {1}", controller.ServiceName, controller.Status); + return -1; + } + + cout.WriteLine("Stopping {0}...", controller.ServiceName); + controller.Stop(); + + if (controller.Status != ServiceControllerStatus.Stopped) + { + cout.WriteLine("Waiting up to 60 seconds for the service to stop (currently: " + controller.Status + ")..."); + controller.WaitForStatus(ServiceControllerStatus.Stopped, TimeSpan.FromSeconds(60)); + } + + if (controller.Status == ServiceControllerStatus.Stopped) + { + cout.WriteLine("Stopped."); + return 0; + } + + cout.WriteLine("The service hasn't stopped successfully."); + return -1; + } + catch (Exception ex) + { + cout.WriteLine(ex.Message); + if (ex.InnerException != null) + cout.WriteLine(ex.InnerException.Message); + return -1; + } + } + } +} + +#endif diff --git a/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs new file mode 100644 index 00000000..baf0320c --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forwarder/TruncateCommand.cs @@ -0,0 +1,51 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using System.Threading.Tasks; +using Seq.Forwarder.Multiplexing; +using SeqCli.Cli; +using SeqCli.Cli.Features; +using Serilog; + +namespace Seq.Forwarder.Cli.Commands +{ + [Command("forwarder", "truncate", "Clear the log buffer contents")] + class TruncateCommand : Command + { + readonly StoragePathFeature _storagePath; + + public TruncateCommand() + { + _storagePath = Enable(); + } + + protected override async Task Run(string[] args) + { + try + { + ActiveLogBufferMap.Truncate(_storagePath.BufferPath); + return 0; + } + catch (Exception ex) + { + await using var logger = new LoggerConfiguration().WriteTo.Console().CreateLogger(); + + logger.Fatal(ex, "Could not truncate log buffer"); + return 1; + } + } + } +} diff --git a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs new file mode 100644 index 00000000..ee717ab5 --- /dev/null +++ b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs @@ -0,0 +1,50 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System; +using System.IO; +using Seq.Forwarder.ServiceProcess; +using Seq.Forwarder.Util; + +namespace Seq.Forwarder.Cli.Commands +{ + [Command("forwarder", "uninstall", "Uninstall the Windows service")] + class UninstallCommand : Command + { + protected override int Run(TextWriter cout) + { + try + { + cout.WriteLine("Uninstalling service..."); + + var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); + var exitCode = CaptiveProcess.Run(sc, $"delete \"{SeqForwarderWindowsService.WindowsServiceName}\"", cout.WriteLine, cout.WriteLine); + if (exitCode != 0) + throw new InvalidOperationException($"The `sc.exe delete` call failed with exit code {exitCode}."); + + cout.WriteLine("Service uninstalled successfully."); + return 0; + } + catch (Exception ex) + { + cout.WriteLine("Could not uninstall the service: " + ex.Message); + return -1; + } + } + } +} + +#endif diff --git a/src/SeqCli/Cli/Features/ListenUriFeature.cs b/src/SeqCli/Cli/Features/ListenUriFeature.cs index 15c5352e..016543ee 100644 --- a/src/SeqCli/Cli/Features/ListenUriFeature.cs +++ b/src/SeqCli/Cli/Features/ListenUriFeature.cs @@ -7,7 +7,7 @@ class ListenUriFeature : CommandFeature public override void Enable(OptionSet options) { options.Add("l=|listen=", - "Set the listen Uri; http://localhost:15341/ is used by default.", + "Set the address `seqcli forwarder` will listen at; http://127.0.0.1:15341/ is used by default.", v => ListenUri = v); } } \ No newline at end of file diff --git a/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs b/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs new file mode 100644 index 00000000..7e7fcd1a --- /dev/null +++ b/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs @@ -0,0 +1,42 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using SeqCli.Cli; + +namespace Seq.Forwarder.Cli.Features +{ + class ServiceCredentialsFeature : CommandFeature + { + public bool IsUsernameSpecified => !string.IsNullOrEmpty(Username); + public bool IsPasswordSpecified => !string.IsNullOrEmpty(Password); + + public string Username { get; set; } = ""; + public string Password { get; set; } = ""; + + public override void Enable(OptionSet options) + { + options.Add("u=|username=", + "The name of a Windows account to run the service under; if not specified the Local System account will be used", + v => Username = v.Trim()); + + options.Add("p=|password=", + "The password for the Windows account to run the service under", + v => Password = v.Trim()); + } + } +} + +#endif diff --git a/src/SeqCli/Cli/Features/StoragePathFeature.cs b/src/SeqCli/Cli/Features/StoragePathFeature.cs index ce363a44..f706eb38 100644 --- a/src/SeqCli/Cli/Features/StoragePathFeature.cs +++ b/src/SeqCli/Cli/Features/StoragePathFeature.cs @@ -48,9 +48,9 @@ static string GetDefaultStorageRoot() static string? TryQueryInstalledStorageRoot() { #if WINDOWS - // if (Seq.Forwarder.Util.ServiceConfiguration.GetServiceStoragePath( - // Seq.Forwarder.ServiceProcess.SeqForwarderWindowsService.WindowsServiceName, out var storage)) - // return storage; + if (Seq.Forwarder.Util.ServiceConfiguration.GetServiceStoragePath( + Seq.Forwarder.ServiceProcess.SeqForwarderWindowsService.WindowsServiceName, out var storage)) + return storage; #endif return null; diff --git a/src/SeqCli/Forwarder/Config/SeqForwarderApiConfig.cs b/src/SeqCli/Forwarder/Config/SeqForwarderApiConfig.cs new file mode 100644 index 00000000..d2e0aaaa --- /dev/null +++ b/src/SeqCli/Forwarder/Config/SeqForwarderApiConfig.cs @@ -0,0 +1,21 @@ +// Copyright 2016-2017 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Seq.Forwarder.Config +{ + class SeqForwarderApiConfig + { + public string ListenUri { get; set; } = "http://localhost:15341"; + } +} diff --git a/src/SeqCli/Forwarder/Config/SeqForwarderConfig.cs b/src/SeqCli/Forwarder/Config/SeqForwarderConfig.cs new file mode 100644 index 00000000..d39abb59 --- /dev/null +++ b/src/SeqCli/Forwarder/Config/SeqForwarderConfig.cs @@ -0,0 +1,95 @@ +// Copyright 2016-2017 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using System.Linq; +using System.Reflection; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Serialization; + +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global + +namespace Seq.Forwarder.Config +{ + class SeqForwarderConfig + { + static JsonSerializerSettings SerializerSettings { get; } = new JsonSerializerSettings + { + ContractResolver = new CamelCasePropertyNamesContractResolver(), + Converters = + { + new StringEnumConverter() + } + }; + + public static SeqForwarderConfig ReadOrInit(string filename, bool includeEnvironmentVariables = true) + { + if (filename == null) throw new ArgumentNullException(nameof(filename)); + + if (!File.Exists(filename)) + { + var config = new SeqForwarderConfig(); + Write(filename, config); + return config; + } + + var content = File.ReadAllText(filename); + var combinedConfig = JsonConvert.DeserializeObject(content, SerializerSettings) + ?? throw new ArgumentException("Configuration content is null."); + + if (includeEnvironmentVariables) + { + // Any Environment Variables overwrite those in the Config File + var envVarConfig = new ConfigurationBuilder().AddEnvironmentVariables("FORWARDER_").Build(); + foreach (var sectionProperty in typeof(SeqForwarderConfig).GetTypeInfo().DeclaredProperties + .Where(p => p.GetMethod != null && p.GetMethod.IsPublic && !p.GetMethod.IsStatic)) + { + foreach (var subGroupProperty in sectionProperty.PropertyType.GetTypeInfo().DeclaredProperties + .Where(p => p.GetMethod != null && p.GetMethod.IsPublic && p.SetMethod != null && p.SetMethod.IsPublic && !p.GetMethod.IsStatic)) + { + var envVarName = sectionProperty.Name.ToUpper() + "_" + subGroupProperty.Name.ToUpper(); + var envVarVal = envVarConfig.GetValue(subGroupProperty.PropertyType, envVarName); + if (envVarVal != null) + { + subGroupProperty.SetValue(sectionProperty.GetValue(combinedConfig), envVarVal); + } + } + } + } + + return combinedConfig; + } + + public static void Write(string filename, SeqForwarderConfig data) + { + if (filename == null) throw new ArgumentNullException(nameof(filename)); + if (data == null) throw new ArgumentNullException(nameof(data)); + + var dir = Path.GetDirectoryName(filename); + if (!Directory.Exists(dir)) + Directory.CreateDirectory(dir!); + + var content = JsonConvert.SerializeObject(data, Formatting.Indented, SerializerSettings); + File.WriteAllText(filename, content); + } + + public SeqForwarderDiagnosticConfig Diagnostics { get; set; } = new SeqForwarderDiagnosticConfig(); + public SeqForwarderOutputConfig Output { get; set; } = new SeqForwarderOutputConfig(); + public SeqForwarderStorageConfig Storage { get; set; } = new SeqForwarderStorageConfig(); + public SeqForwarderApiConfig Api { get; set; } = new SeqForwarderApiConfig(); + } +} diff --git a/src/SeqCli/Forwarder/Config/SeqForwarderDiagnosticConfig.cs b/src/SeqCli/Forwarder/Config/SeqForwarderDiagnosticConfig.cs new file mode 100644 index 00000000..d1bca9f3 --- /dev/null +++ b/src/SeqCli/Forwarder/Config/SeqForwarderDiagnosticConfig.cs @@ -0,0 +1,44 @@ +// Copyright 2016-2017 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using Serilog.Events; + +namespace Seq.Forwarder.Config +{ + public class SeqForwarderDiagnosticConfig + { + public string InternalLogPath { get; set; } = GetDefaultInternalLogPath(); + public LogEventLevel InternalLoggingLevel { get; set; } = LogEventLevel.Information; + public string? InternalLogServerUri { get; set; } + public string? InternalLogServerApiKey { get; set; } + public bool IngestionLogShowDetail { get; set; } + + public static string GetDefaultInternalLogPath() + { + return Path.Combine( +#if WINDOWS + // Common, here, because the service may run as Local Service, which has no obvious home + // directory. + Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), +#else + // Specific to and writable by the current user. + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), +#endif + @"Seq", + "Logs"); + } + } +} diff --git a/src/SeqCli/Forwarder/Config/SeqForwarderOutputConfig.cs b/src/SeqCli/Forwarder/Config/SeqForwarderOutputConfig.cs new file mode 100644 index 00000000..a48bdf76 --- /dev/null +++ b/src/SeqCli/Forwarder/Config/SeqForwarderOutputConfig.cs @@ -0,0 +1,56 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using Newtonsoft.Json; +using Seq.Forwarder.Cryptography; + +// ReSharper disable UnusedMember.Global, AutoPropertyCanBeMadeGetOnly.Global + +namespace Seq.Forwarder.Config +{ + public class SeqForwarderOutputConfig + { + public string ServerUrl { get; set; } = "http://localhost:5341"; + public ulong EventBodyLimitBytes { get; set; } = 256 * 1024; + public ulong RawPayloadLimitBytes { get; set; } = 10 * 1024 * 1024; + public uint? PooledConnectionLifetimeMilliseconds { get; set; } = null; + + const string ProtectedDataPrefix = "pd."; + + public string? ApiKey { get; set; } + + public string? GetApiKey(IStringDataProtector dataProtector) + { + if (string.IsNullOrWhiteSpace(ApiKey)) + return null; + + if (!ApiKey.StartsWith(ProtectedDataPrefix)) + return ApiKey; + + return dataProtector.Unprotect(ApiKey.Substring(ProtectedDataPrefix.Length)); + } + + public void SetApiKey(string? apiKey, IStringDataProtector dataProtector) + { + if (string.IsNullOrWhiteSpace(apiKey)) + { + ApiKey = null; + return; + } + + ApiKey = $"{ProtectedDataPrefix}{dataProtector.Protect(apiKey)}"; + } + } +} diff --git a/src/SeqCli/Forwarder/Config/SeqForwarderStorageConfig.cs b/src/SeqCli/Forwarder/Config/SeqForwarderStorageConfig.cs new file mode 100644 index 00000000..2f713b7d --- /dev/null +++ b/src/SeqCli/Forwarder/Config/SeqForwarderStorageConfig.cs @@ -0,0 +1,21 @@ +// Copyright 2016-2017 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Seq.Forwarder.Config +{ + public class SeqForwarderStorageConfig + { + public ulong BufferSizeBytes { get; set; } = 64 * 1024 * 1024; + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs b/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs new file mode 100644 index 00000000..134eaaa3 --- /dev/null +++ b/src/SeqCli/Forwarder/Cryptography/DpapiMachineScopeDataProtection.cs @@ -0,0 +1,46 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System; +using System.Security.Cryptography; +using System.Text; + +namespace Seq.Forwarder.Cryptography +{ + class DpapiMachineScopeDataProtect : IStringDataProtector + { + public string Unprotect(string @protected) + { + var parts = @protected.Split(new[] { '$' }, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 2) + throw new InvalidOperationException("Encoded data format is invalid."); + + var bytes = Convert.FromBase64String(parts[0]); + var salt = Convert.FromBase64String(parts[1]); + var decoded = ProtectedData.Unprotect(bytes, salt, DataProtectionScope.LocalMachine); + return Encoding.UTF8.GetString(decoded); + } + + public string Protect(string value) + { + var salt = RandomNumberGenerator.GetBytes(16); + var bytes = ProtectedData.Protect(Encoding.UTF8.GetBytes(value), salt, DataProtectionScope.LocalMachine); + return $"{Convert.ToBase64String(bytes)}${Convert.ToBase64String(salt)}"; + } + } +} + +#endif diff --git a/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs b/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs new file mode 100644 index 00000000..24ef61b0 --- /dev/null +++ b/src/SeqCli/Forwarder/Cryptography/IStringDataProtector.cs @@ -0,0 +1,8 @@ +namespace Seq.Forwarder.Cryptography +{ + public interface IStringDataProtector + { + string Protect(string value); + string Unprotect(string @protected); + } +} diff --git a/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs b/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs new file mode 100644 index 00000000..64ef755c --- /dev/null +++ b/src/SeqCli/Forwarder/Cryptography/StringDataProtector.cs @@ -0,0 +1,14 @@ +namespace Seq.Forwarder.Cryptography +{ + static class StringDataProtector + { + public static IStringDataProtector CreatePlatformDefault() + { +#if WINDOWS + return new DpapiMachineScopeDataProtect(); +#else + return new UnprotectedStringData(); +#endif + } + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs b/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs new file mode 100644 index 00000000..b5213375 --- /dev/null +++ b/src/SeqCli/Forwarder/Cryptography/UnprotectedStringData.cs @@ -0,0 +1,22 @@ +#if !WINDOWS + +using Serilog; + +namespace Seq.Forwarder.Cryptography +{ + public class UnprotectedStringData : IStringDataProtector + { + public string Protect(string value) + { + Log.Warning("Data protection is not available on this platform; sensitive values will be stored in plain text"); + return value; + } + + public string Unprotect(string @protected) + { + return @protected; + } + } +} + +#endif diff --git a/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs b/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs new file mode 100644 index 00000000..00797ec0 --- /dev/null +++ b/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs @@ -0,0 +1,49 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using Serilog.Core; +using Serilog.Events; + +namespace Seq.Forwarder.Diagnostics +{ + public class InMemorySink : ILogEventSink + { + readonly int _queueLength; + readonly ConcurrentQueue _queue = new ConcurrentQueue(); + + public InMemorySink(int queueLength) + { + _queueLength = queueLength; + } + + public IEnumerable Read() + { + return _queue.ToArray(); + } + + public void Emit(LogEvent logEvent) + { + if (logEvent == null) throw new ArgumentNullException(nameof(logEvent)); + _queue.Enqueue(logEvent); + + while (_queue.Count > _queueLength) + { + _queue.TryDequeue(out _); + } + } + } +} diff --git a/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs b/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs new file mode 100644 index 00000000..52e5af13 --- /dev/null +++ b/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs @@ -0,0 +1,65 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Net; +using Serilog; +using Serilog.Events; + +namespace Seq.Forwarder.Diagnostics +{ + static class IngestionLog + { + const int Capacity = 100; + + static readonly InMemorySink Sink = new InMemorySink(Capacity); + + public static ILogger Log { get; } + + static IngestionLog() + { + Log = new LoggerConfiguration() + .MinimumLevel.Verbose() + .WriteTo.Sink(Sink) + .WriteTo.Logger(Serilog.Log.Logger) + .CreateLogger(); + } + + public static IEnumerable Read() + { + return Sink.Read(); + } + + public static ILogger ForClient(IPAddress clientHostIP) + { + return Log.ForContext("ClientHostIP", clientHostIP); + } + + public static ILogger ForPayload(IPAddress clientHostIP, string payload) + { + var prefix = CapturePrefix(payload); + return ForClient(clientHostIP) + .ForContext("StartToLog", prefix.Length) + .ForContext("DocumentStart", prefix); + } + + static string CapturePrefix(string line) + { + if (line == null) throw new ArgumentNullException(nameof(line)); + var startToLog = Math.Min(line.Length, 1024); + return line.Substring(0, startToLog); + } + } +} diff --git a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs new file mode 100644 index 00000000..637f4c09 --- /dev/null +++ b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBuffer.cs @@ -0,0 +1,38 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using Seq.Forwarder.Shipper; +using Seq.Forwarder.Storage; + +namespace Seq.Forwarder.Multiplexing +{ + sealed class ActiveLogBuffer : IDisposable + { + public LogShipper Shipper { get; } + public LogBuffer Buffer { get; } + + public ActiveLogBuffer(LogBuffer logBuffer, LogShipper logShipper) + { + Buffer = logBuffer ?? throw new ArgumentNullException(nameof(logBuffer)); + Shipper = logShipper ?? throw new ArgumentNullException(nameof(logShipper)); + } + + public void Dispose() + { + Shipper.Dispose(); + Buffer.Dispose(); + } + } +} diff --git a/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs new file mode 100644 index 00000000..13779570 --- /dev/null +++ b/src/SeqCli/Forwarder/Multiplexing/ActiveLogBufferMap.cs @@ -0,0 +1,235 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using Seq.Forwarder.Config; +using Seq.Forwarder.Cryptography; +using Seq.Forwarder.Storage; +using Seq.Forwarder.Web; +using Serilog; + +namespace Seq.Forwarder.Multiplexing +{ + public class ActiveLogBufferMap : IDisposable + { + const string DataFileName = "data.mdb", LockFileName = "lock.mdb", ApiKeyFileName = ".apikey"; + + readonly ulong _bufferSizeBytes; + readonly SeqForwarderOutputConfig _outputConfig; + readonly ILogShipperFactory _shipperFactory; + readonly IStringDataProtector _dataProtector; + readonly string _bufferPath; + readonly ILogger _log = Log.ForContext(); + + readonly object _sync = new object(); + bool _loaded; + ActiveLogBuffer? _noApiKeyLogBuffer; + readonly Dictionary _buffersByApiKey = new Dictionary(); + + public ActiveLogBufferMap( + string bufferPath, + SeqForwarderStorageConfig storageConfig, + SeqForwarderOutputConfig outputConfig, + ILogShipperFactory logShipperFactory, + IStringDataProtector dataProtector) + { + _bufferSizeBytes = storageConfig.BufferSizeBytes; + _outputConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); + _shipperFactory = logShipperFactory ?? throw new ArgumentNullException(nameof(logShipperFactory)); + _dataProtector = dataProtector ?? throw new ArgumentNullException(nameof(dataProtector)); + _bufferPath = bufferPath ?? throw new ArgumentNullException(nameof(bufferPath)); + } + + // The odd three-stage initialization improves our chances of correctly tearing down the `LightningEnvironment`s within + // `LogBuffer`s in the event of a failure during start-up. See: https://github.com/CoreyKaylor/Lightning.NET/blob/master/src/LightningDB/LightningEnvironment.cs#L252 + public void Load() + { + // At startup, we look for buffers and either delete them if they're empty, or load them + // up if they're not. This garbage collection at start-up is a simplification, + // we might try cleaning up in the background if the gains are worthwhile, although more synchronization + // would be required. + + lock (_sync) + { + if (_loaded) throw new InvalidOperationException("The log buffer map is already loaded."); + + Directory.CreateDirectory(_bufferPath); + + var defaultDataFilePath = Path.Combine(_bufferPath, DataFileName); + if (File.Exists(defaultDataFilePath)) + { + _log.Information("Loading the default log buffer in {Path}", _bufferPath); + var buffer = new LogBuffer(_bufferPath, _bufferSizeBytes); + if (buffer.Peek(0).Length == 0) + { + _log.Information("The default buffer is empty and will be removed until more data is received"); + buffer.Dispose(); + File.Delete(defaultDataFilePath); + var lockFilePath = Path.Combine(_bufferPath, LockFileName); + if (File.Exists(lockFilePath)) + File.Delete(lockFilePath); + } + else + { + _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _outputConfig.GetApiKey(_dataProtector))); + } + } + + foreach (var subfolder in Directory.GetDirectories(_bufferPath)) + { + var encodedApiKeyFilePath = Path.Combine(subfolder, ApiKeyFileName); + if (!File.Exists(encodedApiKeyFilePath)) + { + _log.Information("Folder {Path} does not appear to be a log buffer; skipping", subfolder); + continue; + } + + _log.Information("Loading an API-key specific buffer in {Path}", subfolder); + var apiKey = _dataProtector.Unprotect(File.ReadAllText(encodedApiKeyFilePath)); + + var buffer = new LogBuffer(subfolder, _bufferSizeBytes); + if (buffer.Peek(0).Length == 0) + { + _log.Information("API key-specific buffer in {Path} is empty and will be removed until more data is received", subfolder); + buffer.Dispose(); + Directory.Delete(subfolder, true); + } + else + { + var activeBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, apiKey)); + _buffersByApiKey.Add(apiKey, activeBuffer); + } + } + + _loaded = true; + } + } + + public void Start() + { + lock (_sync) + { + if (!_loaded) throw new InvalidOperationException("The log buffer map is not loaded."); + + foreach (var buffer in OpenBuffers) + { + buffer.Shipper.Start(); + } + } + } + + public void Stop() + { + lock (_sync) + { + // Hard to ensure _loaded is set in all cases, better here to be forgiving and + // permit a clean shut-down. + + foreach (var buffer in OpenBuffers) + { + buffer.Shipper.Stop(); + } + } + } + + public LogBuffer GetLogBuffer(string? apiKey) + { + lock (_sync) + { + if (!_loaded) throw new RequestProcessingException("The forwarder service is starting up.", HttpStatusCode.ServiceUnavailable); + + if (apiKey == null) + { + if (_noApiKeyLogBuffer == null) + { + _log.Information("Creating a new default log buffer in {Path}", _bufferPath); + var buffer = new LogBuffer(_bufferPath, _bufferSizeBytes); + _noApiKeyLogBuffer = new ActiveLogBuffer(buffer, _shipperFactory.Create(buffer, _outputConfig.GetApiKey(_dataProtector))); + _noApiKeyLogBuffer.Shipper.Start(); + } + return _noApiKeyLogBuffer.Buffer; + } + + if (_buffersByApiKey.TryGetValue(apiKey, out var existing)) + return existing.Buffer; + + var subfolder = Path.Combine(_bufferPath, Guid.NewGuid().ToString("n")); + _log.Information("Creating a new API key-specific log buffer in {Path}", subfolder); + Directory.CreateDirectory(subfolder); + File.WriteAllText(Path.Combine(subfolder, ".apikey"), _dataProtector.Protect(apiKey)); + var newBuffer = new LogBuffer(subfolder, _bufferSizeBytes); + var newActiveBuffer = new ActiveLogBuffer(newBuffer, _shipperFactory.Create(newBuffer, apiKey)); + _buffersByApiKey.Add(apiKey, newActiveBuffer); + newActiveBuffer.Shipper.Start(); + return newBuffer; + } + } + + public void Dispose() + { + lock (_sync) + { + foreach (var buffer in OpenBuffers) + { + buffer.Dispose(); + } + } + } + + public void Enumerate(Action action) + { + if (action == null) throw new ArgumentNullException(nameof(action)); + + lock (_sync) + { + foreach (var buffer in OpenBuffers) + { + buffer.Buffer.Enumerate(action); + } + } + } + + public static void Truncate(string bufferPath) + { + DeleteIfExists(Path.Combine(bufferPath, DataFileName)); + DeleteIfExists(Path.Combine(bufferPath, LockFileName)); + foreach (var subdirectory in Directory.GetDirectories(bufferPath)) + { + if (File.Exists(Path.Combine(subdirectory, ApiKeyFileName))) + Directory.Delete(subdirectory, true); + } + } + + static void DeleteIfExists(string filePath) + { + if (File.Exists(filePath)) + File.Delete(filePath); + } + + IEnumerable OpenBuffers + { + get + { + if (_noApiKeyLogBuffer != null) + yield return _noApiKeyLogBuffer; + + foreach (var buffer in _buffersByApiKey.Values) + yield return buffer; + } + } + } +} diff --git a/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs new file mode 100644 index 00000000..4de973ad --- /dev/null +++ b/src/SeqCli/Forwarder/Multiplexing/HttpLogShipperFactory.cs @@ -0,0 +1,41 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Net.Http; +using Seq.Forwarder.Config; +using Seq.Forwarder.Shipper; +using Seq.Forwarder.Storage; + +namespace Seq.Forwarder.Multiplexing +{ + class HttpLogShipperFactory : ILogShipperFactory + { + readonly HttpClient _outputHttpClient; + readonly ServerResponseProxy _serverResponseProxy; + readonly SeqForwarderOutputConfig _outputConfig; + + public HttpLogShipperFactory(ServerResponseProxy serverResponseProxy, SeqForwarderOutputConfig outputConfig, HttpClient outputHttpClient) + { + _outputHttpClient = outputHttpClient; + _serverResponseProxy = serverResponseProxy ?? throw new ArgumentNullException(nameof(serverResponseProxy)); + _outputConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); + } + + public LogShipper Create(LogBuffer logBuffer, string? apiKey) + { + return new HttpLogShipper(logBuffer, apiKey, _outputConfig, _serverResponseProxy, _outputHttpClient); + } + } +} diff --git a/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs new file mode 100644 index 00000000..554324de --- /dev/null +++ b/src/SeqCli/Forwarder/Multiplexing/ILogShipperFactory.cs @@ -0,0 +1,24 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Seq.Forwarder.Shipper; +using Seq.Forwarder.Storage; + +namespace Seq.Forwarder.Multiplexing +{ + public interface ILogShipperFactory + { + LogShipper Create(LogBuffer logBuffer, string? apiKey); + } +} diff --git a/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs b/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs new file mode 100644 index 00000000..f0dd9e44 --- /dev/null +++ b/src/SeqCli/Forwarder/Multiplexing/InertLogShipperFactory.cs @@ -0,0 +1,27 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Seq.Forwarder.Shipper; +using Seq.Forwarder.Storage; + +namespace Seq.Forwarder.Multiplexing +{ + class InertLogShipperFactory : ILogShipperFactory + { + public LogShipper Create(LogBuffer logBuffer, string? apiKey) + { + return new InertLogShipper(); + } + } +} diff --git a/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs b/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs new file mode 100644 index 00000000..86ccc768 --- /dev/null +++ b/src/SeqCli/Forwarder/Multiplexing/ServerResponseProxy.cs @@ -0,0 +1,52 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Collections.Generic; + +namespace Seq.Forwarder.Multiplexing +{ + public class ServerResponseProxy + { + const string EmptyResponse = "{}"; + + readonly object _syncRoot = new object(); + readonly Dictionary _lastResponseByApiKey = new Dictionary(); + string _lastNoApiKeyResponse = EmptyResponse; + + public void SuccessResponseReturned(string? apiKey, string response) + { + lock (_syncRoot) + { + if (apiKey == null) + _lastNoApiKeyResponse = response; + else + _lastResponseByApiKey[apiKey] = response; + } + } + + public string GetResponseText(string? apiKey) + { + lock (_syncRoot) + { + if (apiKey == null) + return _lastNoApiKeyResponse; + + if (_lastResponseByApiKey.TryGetValue(apiKey, out var response)) + return response; + + return EmptyResponse; + } + } + } +} diff --git a/src/SeqCli/Forwarder/Properties/AssemblyInfo.cs b/src/SeqCli/Forwarder/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..da681b10 --- /dev/null +++ b/src/SeqCli/Forwarder/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Seq.Forwarder.Tests")] diff --git a/src/SeqCli/Forwarder/Schema/EventSchema.cs b/src/SeqCli/Forwarder/Schema/EventSchema.cs new file mode 100644 index 00000000..aa6c6f6c --- /dev/null +++ b/src/SeqCli/Forwarder/Schema/EventSchema.cs @@ -0,0 +1,187 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Linq; +using Newtonsoft.Json.Linq; +using Serilog.Parsing; +using Seq.Forwarder.Util; + +namespace Seq.Forwarder.Schema +{ + static class EventSchema + { + static readonly MessageTemplateParser MessageTemplateParser = new MessageTemplateParser(); + + static readonly HashSet ClefReifiedProperties = new HashSet + { + "@t", "@m", "@mt", "@l", "@x", "@i", "@r" + }; + + public static bool FromClefFormat(in int lineNumber, JObject compactFormat, [MaybeNullWhen(false)] out JObject rawFormat, [MaybeNullWhen(true)] out string error) + { + var result = new JObject(); + + var rawTimestamp = compactFormat["@t"]; + if (rawTimestamp == null) + { + error = $"The event on line {lineNumber} does not carry an `@t` timestamp property."; + rawFormat = default; + return false; + } + + if (rawTimestamp.Type != JTokenType.String) + { + error = $"The event on line {lineNumber} has an invalid `@t` timestamp property; the value must be a JSON string."; + rawFormat = default; + return false; + } + + if (!DateTimeOffset.TryParse(rawTimestamp.Value(), out _)) + { + error = $"The timestamp value `{rawTimestamp}` on line {lineNumber} could not be parsed."; + rawFormat = default; + return false; + } + + result.Add("Timestamp", rawTimestamp); + + var properties = new JObject(); + foreach (var property in compactFormat.Properties()) + { + if (property.Name.StartsWith("@@")) + properties.Add(property.Name.Substring(1), property.Value); + else if (!ClefReifiedProperties.Contains(property.Name)) + properties.Add(property.Name, property.Value); + } + + var x = compactFormat["@x"]; + if (x != null) + { + if (x.Type != JTokenType.String) + { + error = $"The event on line {lineNumber} has a non-string `@x` exception property."; + rawFormat = default; + return false; + } + + result.Add("Exception", x); + } + + var l = compactFormat["@l"]; + if (l != null) + { + if (l.Type != JTokenType.String) + { + error = $"The event on line {lineNumber} has a non-string `@l` level property."; + rawFormat = default; + return false; + } + + result.Add("Level", l); + } + + string? message = null; + var m = compactFormat["@m"]; + if (m != null) + { + if (m.Type != JTokenType.String) + { + error = $"The event on line {lineNumber} has a non-string `@m` message property."; + rawFormat = default; + return false; + } + + message = m.Value(); + } + + string? messageTemplate = null; + var mt = compactFormat["@mt"]; + if (mt != null) + { + if (mt.Type != JTokenType.String) + { + error = $"The event on line {lineNumber} has a non-string `@mt` message template property."; + rawFormat = default; + return false; + } + + messageTemplate = mt.Value(); + } + + if (message != null) + { + result.Add("RenderedMessage", message); + } + else if (messageTemplate != null && compactFormat["@r"] is JArray renderingsArray) + { + var template = MessageTemplateParser.Parse(messageTemplate); + var withFormat = template.Tokens.OfType().Where(pt => pt.Format != null); + + // ReSharper disable once PossibleMultipleEnumeration + if (withFormat.Count() == renderingsArray.Count) + { + // ReSharper disable once PossibleMultipleEnumeration + var renderingsByProperty = withFormat + .Zip(renderingsArray, (p, j) => new { p.PropertyName, Format = p.Format!, Rendering = j.Value() }) + .GroupBy(p => p.PropertyName) + .ToDictionary(g => g.Key, g => g.ToDictionaryDistinct(p => p.Format, p => p.Rendering)); + + var renderings = new JObject(); + result.Add("Renderings", renderings); + + foreach (var (property, propertyRenderings) in renderingsByProperty) + { + var byFormat = new JArray(); + renderings.Add(property, byFormat); + + foreach (var (format, rendering) in propertyRenderings) + { + var element = new JObject {{"Format", format}, {"Rendering", rendering}}; + byFormat.Add(element); + } + } + } + } + + messageTemplate ??= message ?? "No template provided"; + result.Add("MessageTemplate", messageTemplate); + + var eventTypeToken = compactFormat["@i"]; + if (eventTypeToken != null) + { + if (eventTypeToken.Type == JTokenType.Integer) + { + result.Add("EventType", uint.Parse(eventTypeToken.Value()!)); + } + else if (eventTypeToken.Type == JTokenType.String) + { + if (uint.TryParse(eventTypeToken.Value(), NumberStyles.HexNumber, + CultureInfo.InvariantCulture, out var eventType)) + { + result.Add("EventType", eventType); + } + else + { + // Seq would calculate a hash value from the string, here. Forwarder will ignore that + // case and preserve the value in an `@i` property for now. + result.Add("@i", eventTypeToken); + } + } + else + { + error = $"The `@i` event type value on line {lineNumber} is not in a string or numeric format."; + rawFormat = default; + return false; + } + } + + if (properties.Count != 0) + result.Add("Properties", properties); + + rawFormat = result; + error = null; + return true; + } + } +} diff --git a/src/SeqCli/Forwarder/SeqForwarderModule.cs b/src/SeqCli/Forwarder/SeqForwarderModule.cs new file mode 100644 index 00000000..d8323510 --- /dev/null +++ b/src/SeqCli/Forwarder/SeqForwarderModule.cs @@ -0,0 +1,86 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Net.Http; +using System.Threading; +using Autofac; +using Seq.Forwarder.Config; +using Seq.Forwarder.Cryptography; +using Seq.Forwarder.Multiplexing; +using Seq.Forwarder.Web.Host; + +namespace Seq.Forwarder +{ + class SeqForwarderModule : Module + { + readonly string _bufferPath; + readonly SeqForwarderConfig _config; + + public SeqForwarderModule(string bufferPath, SeqForwarderConfig config) + { + _bufferPath = bufferPath ?? throw new ArgumentNullException(nameof(bufferPath)); + _config = config ?? throw new ArgumentNullException(nameof(config)); + } + + protected override void Load(ContainerBuilder builder) + { + builder.RegisterType().SingleInstance(); + builder.RegisterType() + .WithParameter("bufferPath", _bufferPath) + .SingleInstance(); + + builder.RegisterType().As(); + builder.RegisterType().SingleInstance(); + + builder.Register(c => + { + var outputConfig = c.Resolve(); + var baseUri = outputConfig.ServerUrl; + if (string.IsNullOrWhiteSpace(baseUri)) + throw new ArgumentException("The destination Seq server URL must be configured in SeqForwarder.json."); + + if (!baseUri.EndsWith("/")) + baseUri += "/"; + + // additional configuration options that require the use of SocketsHttpHandler should be added to + // this expression, using an "or" operator. + + var hasSocketHandlerOption = + (outputConfig.PooledConnectionLifetimeMilliseconds.HasValue); + + if (hasSocketHandlerOption) + { + var httpMessageHandler = new SocketsHttpHandler() + { + PooledConnectionLifetime = (outputConfig.PooledConnectionLifetimeMilliseconds.HasValue) ? TimeSpan.FromMilliseconds(outputConfig.PooledConnectionLifetimeMilliseconds.Value) : Timeout.InfiniteTimeSpan, + }; + + return new HttpClient(httpMessageHandler) { BaseAddress = new Uri(baseUri) }; + } + + return new HttpClient() { BaseAddress = new Uri(baseUri) }; + + }).SingleInstance(); + + builder.RegisterInstance(StringDataProtector.CreatePlatformDefault()); + + builder.RegisterInstance(_config); + builder.RegisterInstance(_config.Api); + builder.RegisterInstance(_config.Diagnostics); + builder.RegisterInstance(_config.Output); + builder.RegisterInstance(_config.Storage); + } + } +} diff --git a/src/SeqCli/Forwarder/ServiceProcess/SeqForwarderWindowsService.cs b/src/SeqCli/Forwarder/ServiceProcess/SeqForwarderWindowsService.cs new file mode 100644 index 00000000..013f67be --- /dev/null +++ b/src/SeqCli/Forwarder/ServiceProcess/SeqForwarderWindowsService.cs @@ -0,0 +1,52 @@ +// Copyright 2020 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System.Net; +using System.ServiceProcess; +using Seq.Forwarder.Web.Host; + +namespace Seq.Forwarder.ServiceProcess +{ + class SeqForwarderWindowsService : ServiceBase + { + readonly ServerService _serverService; + + public static string WindowsServiceName { get; } = "Seq Forwarder"; + + public SeqForwarderWindowsService(ServerService serverService) + { + // Enable TLS 1.2 Support. + // .NET Framework 4.5.2 does not have it enabled by default + ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; + + _serverService = serverService; + + ServiceName = WindowsServiceName; + } + + protected override void OnStart(string[] args) + { + _serverService.Start(); + } + + protected override void OnStop() + { + _serverService.Stop(); + } + } +} + +#endif diff --git a/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs b/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs new file mode 100644 index 00000000..84c32f6c --- /dev/null +++ b/src/SeqCli/Forwarder/Shipper/ExponentialBackoffConnectionSchedule.cs @@ -0,0 +1,74 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace Seq.Forwarder.Shipper +{ + class ExponentialBackoffConnectionSchedule + { + static readonly TimeSpan MinimumBackoffPeriod = TimeSpan.FromSeconds(5); + static readonly TimeSpan MaximumBackoffInterval = TimeSpan.FromMinutes(10); + + readonly TimeSpan _period; + + int _failuresSinceSuccessfulConnection; + + public ExponentialBackoffConnectionSchedule(TimeSpan period) + { + if (period < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(period), "The connection retry period must be a positive timespan"); + + _period = period; + } + + public void MarkSuccess() + { + _failuresSinceSuccessfulConnection = 0; + } + + public void MarkFailure() + { + ++_failuresSinceSuccessfulConnection; + } + + public bool LastConnectionFailed => _failuresSinceSuccessfulConnection != 0; + + public TimeSpan NextInterval + { + get + { + // Available, and first failure, just try the batch interval + if (_failuresSinceSuccessfulConnection <= 1) return _period; + + // Second failure, start ramping up the interval - first 2x, then 4x, ... + var backoffFactor = Math.Pow(2, (_failuresSinceSuccessfulConnection - 1)); + + // If the period is ridiculously short, give it a boost so we get some + // visible backoff. + var backoffPeriod = Math.Max(_period.Ticks, MinimumBackoffPeriod.Ticks); + + // The "ideal" interval + var backedOff = (long)(backoffPeriod * backoffFactor); + + // Capped to the maximum interval + var cappedBackoff = Math.Min(MaximumBackoffInterval.Ticks, backedOff); + + // Unless that's shorter than the base interval, in which case we'll just apply the period + var actual = Math.Max(_period.Ticks, cappedBackoff); + + return TimeSpan.FromTicks(actual); + } + } + } +} diff --git a/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs b/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs new file mode 100644 index 00000000..5ebe7eb1 --- /dev/null +++ b/src/SeqCli/Forwarder/Shipper/HttpLogShipper.cs @@ -0,0 +1,251 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using Seq.Forwarder.Config; +using Seq.Forwarder.Storage; +using Serilog; +using System.Threading.Tasks; +using Seq.Forwarder.Multiplexing; +using Seq.Forwarder.Util; + +namespace Seq.Forwarder.Shipper +{ + sealed class HttpLogShipper : LogShipper + { + const string BulkUploadResource = "api/events/raw"; + + readonly string? _apiKey; + readonly LogBuffer _logBuffer; + readonly SeqForwarderOutputConfig _outputConfig; + readonly HttpClient _httpClient; + readonly ExponentialBackoffConnectionSchedule _connectionSchedule; + readonly ServerResponseProxy _serverResponseProxy; + DateTime _nextRequiredLevelCheck; + + readonly object _stateLock = new object(); + readonly Timer _timer; + bool _started; + + volatile bool _unloading; + + static readonly TimeSpan QuietWaitPeriod = TimeSpan.FromSeconds(2), MaximumConnectionInterval = TimeSpan.FromMinutes(2); + + public HttpLogShipper(LogBuffer logBuffer, string? apiKey, SeqForwarderOutputConfig outputConfig, ServerResponseProxy serverResponseProxy, HttpClient outputHttpClient) + { + _apiKey = apiKey; + _httpClient = outputHttpClient ?? throw new ArgumentNullException(nameof(outputHttpClient)); + _logBuffer = logBuffer ?? throw new ArgumentNullException(nameof(logBuffer)); + _outputConfig = outputConfig ?? throw new ArgumentNullException(nameof(outputConfig)); + _serverResponseProxy = serverResponseProxy ?? throw new ArgumentNullException(nameof(serverResponseProxy)); + _connectionSchedule = new ExponentialBackoffConnectionSchedule(QuietWaitPeriod); + _timer = new Timer(s => OnTick()); + } + + public override void Start() + { + lock (_stateLock) + { + if (_started) + throw new InvalidOperationException("The shipper has already started."); + + if (_unloading) + throw new InvalidOperationException("The shipper is unloading."); + + Log.Information("Log shipper started, events will be dispatched to {ServerUrl}", _outputConfig.ServerUrl); + + _nextRequiredLevelCheck = DateTime.UtcNow.Add(MaximumConnectionInterval); + _started = true; + SetTimer(); + } + } + + public override void Stop() + { + lock (_stateLock) + { + if (_unloading) + return; + + _unloading = true; + + if (!_started) + return; + } + + var wh = new ManualResetEvent(false); + if (_timer.Dispose(wh)) + wh.WaitOne(); + } + + public override void Dispose() + { + Stop(); + } + + void SetTimer() + { + _timer.Change(_connectionSchedule.NextInterval, Timeout.InfiniteTimeSpan); + } + + void OnTick() + { + OnTickAsync().Wait(); + } + + async Task OnTickAsync() + { + try + { + var sendingSingles = 0; + do + { + var available = _logBuffer.Peek((int)_outputConfig.RawPayloadLimitBytes); + if (available.Length == 0) + { + if (DateTime.UtcNow < _nextRequiredLevelCheck || _connectionSchedule.LastConnectionFailed) + { + // For whatever reason, there's nothing waiting to send. This means we should try connecting again at the + // regular interval, so mark the attempt as successful. + _connectionSchedule.MarkSuccess(); + break; + } + } + + MakePayload(available, sendingSingles > 0, out Stream payload, out ulong lastIncluded); + + var content = new StreamContent(new UnclosableStreamWrapper(payload)); + content.Headers.ContentType = new MediaTypeHeaderValue("application/json") + { + CharSet = Encoding.UTF8.WebName + }; + + if (_apiKey != null) + { + content.Headers.Add(SeqApi.ApiKeyHeaderName, _apiKey); + } + + var result = await _httpClient.PostAsync(BulkUploadResource, content); + if (result.IsSuccessStatusCode) + { + _connectionSchedule.MarkSuccess(); + _logBuffer.Dequeue(lastIncluded); + if (sendingSingles > 0) + sendingSingles--; + + _serverResponseProxy.SuccessResponseReturned(_apiKey, await result.Content.ReadAsStringAsync()); + _nextRequiredLevelCheck = DateTime.UtcNow.Add(MaximumConnectionInterval); + } + else if (result.StatusCode == HttpStatusCode.BadRequest || + result.StatusCode == HttpStatusCode.RequestEntityTooLarge) + { + // The connection attempt was successful - the payload we sent was the problem. + _connectionSchedule.MarkSuccess(); + + if (sendingSingles != 0) + { + payload.Position = 0; + var payloadText = await new StreamReader(payload, Encoding.UTF8).ReadToEndAsync(); + Log.Error("HTTP shipping failed with {StatusCode}: {Result}; payload was {InvalidPayload}", result.StatusCode, await result.Content.ReadAsStringAsync(), payloadText); + _logBuffer.Dequeue(lastIncluded); + sendingSingles = 0; + } + else + { + // Unscientific (should "binary search" in batches) but sending the next + // hundred events singly should flush out the problematic one. + sendingSingles = 100; + } + } + else + { + _connectionSchedule.MarkFailure(); + Log.Error("Received failed HTTP shipping result {StatusCode}: {Result}", result.StatusCode, await result.Content.ReadAsStringAsync()); + break; + } + } + while (true); + } + catch (HttpRequestException hex) + { + Log.Warning(hex, "HTTP request failed when sending a batch from the log shipper"); + _connectionSchedule.MarkFailure(); + } + catch (Exception ex) + { + Log.Error(ex, "Exception while sending a batch from the log shipper"); + _connectionSchedule.MarkFailure(); + } + finally + { + lock (_stateLock) + { + if (!_unloading) + SetTimer(); + } + } + } + + void MakePayload(LogBufferEntry[] entries, bool oneOnly, out Stream utf8Payload, out ulong lastIncluded) + { + if (entries == null) throw new ArgumentNullException(nameof(entries)); + lastIncluded = 0; + + var raw = new MemoryStream(); + var content = new StreamWriter(raw, Encoding.UTF8); + content.Write("{\"Events\":["); + content.Flush(); + var contentRemainingBytes = (int) _outputConfig.RawPayloadLimitBytes - 13; // Includes closing delims + + var delimStart = ""; + foreach (var logBufferEntry in entries) + { + if ((ulong)logBufferEntry.Value.Length > _outputConfig.EventBodyLimitBytes) + { + Log.Information("Oversized event will be skipped, {Payload}", Encoding.UTF8.GetString(logBufferEntry.Value)); + lastIncluded = logBufferEntry.Key; + continue; + } + + // lastIncluded indicates we've added at least one event + if (lastIncluded != 0 && contentRemainingBytes - (delimStart.Length + logBufferEntry.Value.Length) < 0) + break; + + content.Write(delimStart); + content.Flush(); + contentRemainingBytes -= delimStart.Length; + + raw.Write(logBufferEntry.Value, 0, logBufferEntry.Value.Length); + contentRemainingBytes -= logBufferEntry.Value.Length; + + lastIncluded = logBufferEntry.Key; + + delimStart = ","; + if (oneOnly) + break; + } + + content.Write("]}"); + content.Flush(); + raw.Position = 0; + utf8Payload = raw; + } + } +} diff --git a/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs b/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs new file mode 100644 index 00000000..164a2939 --- /dev/null +++ b/src/SeqCli/Forwarder/Shipper/InertLogShipper.cs @@ -0,0 +1,31 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Seq.Forwarder.Shipper +{ + class InertLogShipper : LogShipper + { + public override void Start() + { + } + + public override void Stop() + { + } + + public override void Dispose() + { + } + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Shipper/LogShipper.cs b/src/SeqCli/Forwarder/Shipper/LogShipper.cs new file mode 100644 index 00000000..ac8f5157 --- /dev/null +++ b/src/SeqCli/Forwarder/Shipper/LogShipper.cs @@ -0,0 +1,25 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace Seq.Forwarder.Shipper +{ + public abstract class LogShipper : IDisposable + { + public abstract void Start(); + public abstract void Stop(); + public abstract void Dispose(); + } +} diff --git a/src/SeqCli/Forwarder/Shipper/SeqApi.cs b/src/SeqCli/Forwarder/Shipper/SeqApi.cs new file mode 100644 index 00000000..330dc3b8 --- /dev/null +++ b/src/SeqCli/Forwarder/Shipper/SeqApi.cs @@ -0,0 +1,21 @@ +// Copyright © Datalust Pty Ltd and Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Seq.Forwarder.Shipper +{ + static class SeqApi + { + public const string ApiKeyHeaderName = "X-Seq-ApiKey"; + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Storage/LogBuffer.cs b/src/SeqCli/Forwarder/Storage/LogBuffer.cs new file mode 100644 index 00000000..bac905ea --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/LogBuffer.cs @@ -0,0 +1,280 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Serilog; + +namespace Seq.Forwarder.Storage +{ + public class LogBuffer : IDisposable + { + readonly ulong _bufferSizeBytes; + // readonly LightningEnvironment _env; + readonly object _sync = new object(); + bool _isDisposed; + ulong _nextId = 0, _entries = 0, _writtenSinceRotateCheck; + + public LogBuffer(string bufferPath, ulong bufferSizeBytes) + { + _bufferSizeBytes = bufferSizeBytes; + if (bufferPath == null) throw new ArgumentNullException(nameof(bufferPath)); + + // _env = new LightningEnvironment(bufferPath) + // { + // // Sparse; we'd hope fragmentation never gets this bad... + // MapSize = (long) bufferSizeBytes*10 + // }; + // + // _env.Open(); + // + // using (var tx = _env.BeginTransaction()) + // using (var db = tx.OpenDatabase()) + // { + // using (var cur = tx.CreateCursor(db)) + // { + // if (!cur.MoveToLast()) + // { + // _nextId = 1; + // } + // else + // { + // var current = cur.GetCurrent(); + // _nextId = ByteKeyToULongKey(current.Key) + 1; + // _entries = (ulong) tx.GetEntriesCount(db); + // } + // } + // } + + Log.Information("Log buffer open on {BufferPath}; {Entries} entries, next key will be {NextId}", bufferPath, _entries, _nextId); + } + + public void Dispose() + { + lock (_sync) + { + if (!_isDisposed) + { + _isDisposed = true; + // _env.Dispose(); + } + } + } + + public void Enqueue(byte[][] values) + { + if (values == null) throw new ArgumentNullException(nameof(values)); + + lock (_sync) + { + RequireNotDisposed(); + + // var totalPayloadWritten = 0UL; + // + // using (var tx = _env.BeginTransaction()) + // using (var db = tx.OpenDatabase()) + // { + // foreach (var v in values) + // { + // if (v == null) throw new ArgumentException("Value array may not contain null."); + // + // tx.Put(db, ULongKeyToByteKey(_nextId++), v); + // totalPayloadWritten += (ulong) v.Length; + // } + // + // tx.Commit(); + // _entries += (ulong) values.Length; + // _writtenSinceRotateCheck += totalPayloadWritten; + // } + + RotateIfRequired(); + } + } + + void RotateIfRequired() + { + if (_writtenSinceRotateCheck < _bufferSizeBytes/10) + return; + + _writtenSinceRotateCheck = 0; + // + // using (var tx = _env.BeginTransaction()) + // using (var db = tx.OpenDatabase()) + // { + // int err; + // if (0 != (err = Lmdb.mdb_env_info(_env.Handle(), out var estat))) + // throw new Exception(Lmdb.mdb_strerror(err)); + // + // MDBStat stat; + // if (0 != (err = Lmdb.mdb_stat(tx.Handle(), db.Handle(), out stat))) + // throw new Exception(Lmdb.mdb_strerror(err)); + // + // // http://www.openldap.org/lists/openldap-technical/201303/msg00145.html + // // 1) MDB_stat gives you the page size. + // // 2) MDB_envinfo tells the mapsize and the last_pgno.If you divide mapsize + // // by pagesize you'll get max pgno. The MAP_FULL error is returned when last_pgno reaches max pgno. + // + // var targetPages = _bufferSizeBytes/stat.ms_psize; + // if ((ulong) estat.me_last_pgno < targetPages && (double) (ulong) estat.me_last_pgno/targetPages < 0.75) + // return; + // + // var count = tx.GetEntriesCount(db); + // if (count == 0) + // { + // Log.Warning("Attempting to rotate buffer but no events are present"); + // return; + // } + // + // var toPurge = Math.Max(count / 4, 1); + // Log.Warning("Buffer is full; dropping {ToPurge} events to make room for new ones", + // toPurge); + // + // using (var cur = tx.CreateCursor(db)) + // { + // cur.MoveToFirst(); + // + // for (var i = 0; i < toPurge; ++i) + // { + // cur.Delete(); + // cur.MoveNext(); + // } + // } + // + // tx.Commit(); + // } + } + + public LogBufferEntry[] Peek(int maxValueBytesHint) + { + lock (_sync) + { + RequireNotDisposed(); + + var entries = new List(); + // + // using (var tx = _env.BeginTransaction(TransactionBeginFlags.ReadOnly)) + // using (var db = tx.OpenDatabase()) + // { + // using (var cur = tx.CreateCursor(db)) + // { + // if (cur.MoveToFirst()) + // { + // var entriesBytes = 0; + // + // do + // { + // var current = cur.GetCurrent(); + // var entry = new LogBufferEntry + // { + // Key = ByteKeyToULongKey(current.Key), + // Value = current.Value + // }; + // + // entriesBytes += entry.Value.Length; + // if (entries.Count != 0 && entriesBytes > maxValueBytesHint) + // break; + // + // entries.Add(entry); + // + // } while (cur.MoveNext()); + // } + // } + // } + + return entries.ToArray(); + } + } + + public void Dequeue(ulong toKey) + { + lock (_sync) + { + RequireNotDisposed(); + + // ulong deleted = 0; + // + // using (var tx = _env.BeginTransaction()) + // using (var db = tx.OpenDatabase()) + // { + // using (var cur = tx.CreateCursor(db)) + // { + // if (cur.MoveToFirst()) + // { + // do + // { + // var current = cur.GetCurrent(); + // if (ByteKeyToULongKey(current.Key) > toKey) + // break; + // + // cur.Delete(); + // deleted++; + // } while (cur.MoveNext()); + // } + // } + // + // tx.Commit(); + // _entries -= deleted; + // } + } + } + + void RequireNotDisposed() + { + if (_isDisposed) + throw new ObjectDisposedException(typeof(LogBuffer).FullName); + } + + static ulong ByteKeyToULongKey(byte[] key) + { + var copy = new byte[key.Length]; + for (var i = 0; i < key.Length; ++i) + copy[copy.Length - (i + 1)] = key[i]; + + return BitConverter.ToUInt64(copy, 0); + } + + static byte[] ULongKeyToByteKey(ulong key) + { + var k = BitConverter.GetBytes(key); + Array.Reverse(k); + return k; + } + + public void Enumerate(Action action) + { + if (action == null) throw new ArgumentNullException(nameof(action)); + + lock (_sync) + { + RequireNotDisposed(); + + // using (var tx = _env.BeginTransaction(TransactionBeginFlags.ReadOnly)) + // using (var db = tx.OpenDatabase()) + // { + // using (var cur = tx.CreateCursor(db)) + // { + // if (cur.MoveToFirst()) + // { + // do + // { + // var current = cur.GetCurrent(); + // action(ByteKeyToULongKey(current.Key), current.Value); + // } while (cur.MoveNext()); + // } + // } + // } + } + } + } +} diff --git a/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs b/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs new file mode 100644 index 00000000..464a7175 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs @@ -0,0 +1,24 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ReSharper disable InconsistentNaming + +namespace Seq.Forwarder.Storage +{ + public struct LogBufferEntry + { + public ulong Key; + public byte[] Value; + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Util/AccountRightsHelper.cs b/src/SeqCli/Forwarder/Util/AccountRightsHelper.cs new file mode 100644 index 00000000..6b73caf8 --- /dev/null +++ b/src/SeqCli/Forwarder/Util/AccountRightsHelper.cs @@ -0,0 +1,193 @@ +// Original interop code copyright Corinna John +// Used under CPOL. http://www.codeproject.com/info/cpol10.aspx +// http://www.codeproject.com/Articles/4863/LSA-Functions-Privileges-and-Impersonation +// Modified and reformatted. + +#if WINDOWS + +using System; +using System.Runtime.InteropServices; +using System.Text; + +// ReSharper disable FieldCanBeMadeReadOnly.Local + +namespace Seq.Forwarder.Util +{ + public static class AccountRightsHelper + { + [DllImport("advapi32.dll", PreserveSig = true)] + private static extern UInt32 LsaOpenPolicy( + ref LSA_UNICODE_STRING SystemName, + ref LSA_OBJECT_ATTRIBUTES ObjectAttributes, + Int32 DesiredAccess, + out IntPtr PolicyHandle + ); + + [DllImport("advapi32.dll", SetLastError = true, PreserveSig = true)] + private static extern long LsaAddAccountRights( + IntPtr PolicyHandle, + IntPtr AccountSid, + LSA_UNICODE_STRING[] UserRights, + long CountOfRights); + + [DllImport("advapi32")] + public static extern void FreeSid(IntPtr pSid); + + [DllImport("advapi32.dll", CharSet = CharSet.Auto, SetLastError = true, PreserveSig = true)] + private static extern bool LookupAccountName( + string lpSystemName, string lpAccountName, + IntPtr psid, + ref int cbsid, + StringBuilder domainName, ref int cbdomainLength, ref int use); + + [DllImport("advapi32.dll")] + private static extern long LsaClose(IntPtr ObjectHandle); + + [DllImport("kernel32.dll")] + private static extern int GetLastError(); + + [DllImport("advapi32.dll")] + private static extern long LsaNtStatusToWinError(long status); + + // define the structures + + [StructLayout(LayoutKind.Sequential)] + private struct LSA_UNICODE_STRING + { + public UInt16 Length; + public UInt16 MaximumLength; + public IntPtr Buffer; + } + + [StructLayout(LayoutKind.Sequential)] + private struct LSA_OBJECT_ATTRIBUTES + { + public int Length; + public IntPtr RootDirectory; + public LSA_UNICODE_STRING ObjectName; + public UInt32 Attributes; + public IntPtr SecurityDescriptor; + public IntPtr SecurityQualityOfService; + } + + // enum all policies + + [Flags] + private enum LSA_AccessPolicy : long + { + POLICY_VIEW_LOCAL_INFORMATION = 0x00000001L, + POLICY_VIEW_AUDIT_INFORMATION = 0x00000002L, + POLICY_GET_PRIVATE_INFORMATION = 0x00000004L, + POLICY_TRUST_ADMIN = 0x00000008L, + POLICY_CREATE_ACCOUNT = 0x00000010L, + POLICY_CREATE_SECRET = 0x00000020L, + POLICY_CREATE_PRIVILEGE = 0x00000040L, + POLICY_SET_DEFAULT_QUOTA_LIMITS = 0x00000080L, + POLICY_SET_AUDIT_REQUIREMENTS = 0x00000100L, + POLICY_AUDIT_LOG_ADMIN = 0x00000200L, + POLICY_SERVER_ADMIN = 0x00000400L, + POLICY_LOOKUP_NAMES = 0x00000800L, + POLICY_NOTIFICATION = 0x00001000L + } + + /// Adds a privilege to an account + /// Name of an account - "domain\account" or only "account" + /// Name ofthe privilege + /// The windows error code returned by LsaAddAccountRights + static long SetRight(string accountName, string privilegeName) + { + long winErrorCode; //contains the last error + + //pointer an size for the SID + IntPtr sid = IntPtr.Zero; + int sidSize = 0; + //StringBuilder and size for the domain name + StringBuilder domainName = new StringBuilder(); + int nameSize = 0; + //account-type variable for lookup + int accountType = 0; + + //get required buffer size + LookupAccountName(String.Empty, accountName, sid, ref sidSize, domainName, ref nameSize, ref accountType); + + //allocate buffers + domainName = new StringBuilder(nameSize); + sid = Marshal.AllocHGlobal(sidSize); + + //lookup the SID for the account + bool result = LookupAccountName(String.Empty, accountName, sid, ref sidSize, domainName, ref nameSize, ref accountType); + + if (!result) + { + winErrorCode = GetLastError(); + } + else + { + //initialize an empty unicode-string + LSA_UNICODE_STRING systemName = new LSA_UNICODE_STRING(); + //combine all policies + int access = (int)( + LSA_AccessPolicy.POLICY_AUDIT_LOG_ADMIN | + LSA_AccessPolicy.POLICY_CREATE_ACCOUNT | + LSA_AccessPolicy.POLICY_CREATE_PRIVILEGE | + LSA_AccessPolicy.POLICY_CREATE_SECRET | + LSA_AccessPolicy.POLICY_GET_PRIVATE_INFORMATION | + LSA_AccessPolicy.POLICY_LOOKUP_NAMES | + LSA_AccessPolicy.POLICY_NOTIFICATION | + LSA_AccessPolicy.POLICY_SERVER_ADMIN | + LSA_AccessPolicy.POLICY_SET_AUDIT_REQUIREMENTS | + LSA_AccessPolicy.POLICY_SET_DEFAULT_QUOTA_LIMITS | + LSA_AccessPolicy.POLICY_TRUST_ADMIN | + LSA_AccessPolicy.POLICY_VIEW_AUDIT_INFORMATION | + LSA_AccessPolicy.POLICY_VIEW_LOCAL_INFORMATION + ); + + //initialize a pointer for the policy handle + IntPtr policyHandle; + + //these attributes are not used, but LsaOpenPolicy wants them to exists + LSA_OBJECT_ATTRIBUTES ObjectAttributes = new LSA_OBJECT_ATTRIBUTES(); + ObjectAttributes.Length = 0; + ObjectAttributes.RootDirectory = IntPtr.Zero; + ObjectAttributes.Attributes = 0; + ObjectAttributes.SecurityDescriptor = IntPtr.Zero; + ObjectAttributes.SecurityQualityOfService = IntPtr.Zero; + + //get a policy handle + uint resultPolicy = LsaOpenPolicy(ref systemName, ref ObjectAttributes, access, out policyHandle); + winErrorCode = LsaNtStatusToWinError(resultPolicy); + + if (winErrorCode == 0) + { + //Now that we have the SID an the policy, + //we can add rights to the account. + + //initialize an unicode-string for the privilege name + LSA_UNICODE_STRING[] userRights = new LSA_UNICODE_STRING[1]; + userRights[0] = new LSA_UNICODE_STRING(); + userRights[0].Buffer = Marshal.StringToHGlobalUni(privilegeName); + userRights[0].Length = (UInt16)(privilegeName.Length * UnicodeEncoding.CharSize); + userRights[0].MaximumLength = (UInt16)((privilegeName.Length + 1) * UnicodeEncoding.CharSize); + + //add the right to the account + long res = LsaAddAccountRights(policyHandle, sid, userRights, 1); + winErrorCode = LsaNtStatusToWinError(res); + LsaClose(policyHandle); + } + + FreeSid(sid); + } + + return winErrorCode; + } + + public static void EnsureServiceLogOnRights(string accountName) + { + var err = SetRight(accountName, "SeServiceLogonRight"); + if (err != 0) + Marshal.ThrowExceptionForHR(Marshal.GetHRForLastWin32Error()); + } + } +} + +#endif diff --git a/src/SeqCli/Forwarder/Util/CaptiveProcess.cs b/src/SeqCli/Forwarder/Util/CaptiveProcess.cs new file mode 100644 index 00000000..dc12482e --- /dev/null +++ b/src/SeqCli/Forwarder/Util/CaptiveProcess.cs @@ -0,0 +1,82 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics; +using System.Threading; + +namespace Seq.Forwarder.Util +{ + public static class CaptiveProcess + { + public static int Run( + string fullExePath, + string? args = null, + Action? writeStdout = null, + Action? writeStderr = null, + string? workingDirectory = null) + { + if (fullExePath == null) throw new ArgumentNullException(nameof(fullExePath)); + + args ??= ""; + writeStdout ??= delegate { }; + writeStderr ??= delegate { }; + + var startInfo = new ProcessStartInfo + { + UseShellExecute = false, + RedirectStandardError = true, + RedirectStandardOutput = true, + WindowStyle = ProcessWindowStyle.Hidden, + CreateNoWindow = true, + ErrorDialog = false, + FileName = fullExePath, + Arguments = args + }; + + if (!string.IsNullOrEmpty(workingDirectory)) + startInfo.WorkingDirectory = workingDirectory; + + using var process = Process.Start(startInfo)!; + using var outputComplete = new ManualResetEvent(false); + using var errorComplete = new ManualResetEvent(false); + // ReSharper disable AccessToDisposedClosure + + process.OutputDataReceived += (_, e) => + { + if (e.Data == null) + outputComplete.Set(); + else + writeStdout(e.Data); + }; + process.BeginOutputReadLine(); + + process.ErrorDataReceived += (_, e) => + { + if (e.Data == null) + errorComplete.Set(); + else + writeStderr(e.Data); + }; + process.BeginErrorReadLine(); + + process.WaitForExit(); + + outputComplete.WaitOne(); + errorComplete.WaitOne(); + + return process.ExitCode; + } + } +} diff --git a/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs b/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs new file mode 100644 index 00000000..612bc684 --- /dev/null +++ b/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; + +namespace Seq.Forwarder.Util +{ + static class EnumerableExtensions + { + public static Dictionary ToDictionaryDistinct( + this IEnumerable enumerable, Func keySelector, Func valueSelector) + where TKey: notnull + { + var result = new Dictionary(); + foreach (var e in enumerable) + { + result[keySelector(e)] = valueSelector(e); + } + return result; + } + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs b/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs new file mode 100644 index 00000000..9e94295a --- /dev/null +++ b/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs @@ -0,0 +1,20 @@ +namespace Seq.Forwarder.Util +{ + static class ExecutionEnvironment + { + public static bool SupportsStandardIO => !IsRunningAsWindowsService; + + static bool IsRunningAsWindowsService + { + get + { +#if WINDOWS + var parent = WindowsProcess.GetParentProcess(); + return parent?.ProcessName == "services"; +#else + return false; +#endif + } + } + } +} diff --git a/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs b/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs new file mode 100644 index 00000000..2fda1fd5 --- /dev/null +++ b/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs @@ -0,0 +1,111 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if WINDOWS + +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.ServiceProcess; +using System.Text; + +namespace Seq.Forwarder.Util +{ + public static class ServiceConfiguration + { + public static bool GetServiceBinaryPath(ServiceController controller, TextWriter cout, [MaybeNullWhen(false)] out string path) + { + var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); + + var config = new StringBuilder(); + if (0 != CaptiveProcess.Run(sc, "qc \"" + controller.ServiceName + "\"", l => config.AppendLine(l), cout.WriteLine)) + { + cout.WriteLine("Could not query service path; ignoring."); + path = null; + return false; + } + + var lines = config.ToString() + .Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries) + .Select(l => l.Trim()); + + var line = lines + .SingleOrDefault(l => l.StartsWith("BINARY_PATH_NAME : ")); + + if (line == null) + { + cout.WriteLine("No existing binary path could be determined."); + path = null; + return false; + } + + path = line.Replace("BINARY_PATH_NAME : ", ""); + return true; + } + + static bool GetServiceCommandLine(string serviceName, TextWriter cout, [MaybeNullWhen(false)] out string path) + { + if (serviceName == null) throw new ArgumentNullException(nameof(serviceName)); + if (cout == null) throw new ArgumentNullException(nameof(cout)); + + var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); + + var config = new StringBuilder(); + if (0 != CaptiveProcess.Run(sc, "qc \"" + serviceName + "\"", l => config.AppendLine(l), cout.WriteLine)) + { + cout.WriteLine("Could not query service path; ignoring."); + path = null; + return false; + } + + var lines = config.ToString() + .Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries) + .Select(l => l.Trim()); + + var line = lines + .SingleOrDefault(l => l.StartsWith("BINARY_PATH_NAME : ")); + + if (line == null) + { + cout.WriteLine("No existing binary path could be determined."); + path = null; + return false; + } + + path = line.Replace("BINARY_PATH_NAME : ", ""); + return true; + } + + public static bool GetServiceStoragePath(string serviceName, out string? storage) + { + if (serviceName == null) throw new ArgumentNullException(nameof(serviceName)); + + if (GetServiceCommandLine(serviceName, new StringWriter(), out var binpath) && + binpath.Contains("--storage=\"")) + { + var start = binpath.IndexOf("--storage=\"", StringComparison.Ordinal) + 11; + var chop = binpath.Substring(start); + storage = chop.Substring(0, chop.IndexOf('"')); + return true; + } + + storage = null; + return false; + } + } +} + +#endif + diff --git a/src/SeqCli/Forwarder/Util/UnclosableStreamWrapper.cs b/src/SeqCli/Forwarder/Util/UnclosableStreamWrapper.cs new file mode 100644 index 00000000..ce86ea12 --- /dev/null +++ b/src/SeqCli/Forwarder/Util/UnclosableStreamWrapper.cs @@ -0,0 +1,60 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; + +namespace Seq.Forwarder.Util +{ + class UnclosableStreamWrapper : Stream + { + readonly Stream _stream; + + public UnclosableStreamWrapper(Stream stream) + { + _stream = stream ?? throw new ArgumentNullException(nameof(stream)); + } + + public override void Flush() + { + _stream.Flush(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + return _stream.Seek(offset, origin); + } + + public override void SetLength(long value) + { + _stream.SetLength(value); + } + + public override int Read(byte[] buffer, int offset, int count) + { + return _stream.Read(buffer, offset, count); + } + + public override void Write(byte[] buffer, int offset, int count) + { + _stream.Write(buffer, offset, count); + } + + public override bool CanRead => _stream.CanRead; + public override bool CanSeek => _stream.CanSeek; + public override bool CanWrite => _stream.CanWrite; + public override long Length => _stream.Length; + public override long Position { get { return _stream.Position; } set { _stream.Position = value; } } + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Util/WindowsProcess.cs b/src/SeqCli/Forwarder/Util/WindowsProcess.cs new file mode 100644 index 00000000..98a20930 --- /dev/null +++ b/src/SeqCli/Forwarder/Util/WindowsProcess.cs @@ -0,0 +1,51 @@ +#if WINDOWS + +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.Runtime.InteropServices; +using Serilog; + +// ReSharper disable once InconsistentNaming + +namespace Seq.Forwarder.Util +{ + static class WindowsProcess + { + [StructLayout(LayoutKind.Sequential)] + readonly struct PROCESS_BASIC_INFORMATION + { + readonly IntPtr _reserved1; + readonly IntPtr _pebBaseAddress; + readonly IntPtr _reserved2_0; + readonly IntPtr _reserved2_1; + readonly IntPtr _uniqueProcessId; + public readonly IntPtr InheritedFromUniqueProcessId; + } + + [DllImport("ntdll.dll")] + static extern int NtQueryInformationProcess(IntPtr processHandle, int processInformationClass, ref PROCESS_BASIC_INFORMATION processInformation, int processInformationLength, out int returnLength); + + public static Process? GetParentProcess() + { + var currentProcess = Process.GetCurrentProcess(); + + var pbi = new PROCESS_BASIC_INFORMATION(); + var status = NtQueryInformationProcess(currentProcess.Handle, 0, ref pbi, Marshal.SizeOf(pbi), out _); + if (status != 0) + throw new Win32Exception(status); + + try + { + return Process.GetProcessById(pbi.InheritedFromUniqueProcessId.ToInt32()); + } + catch (Exception ex) + { + Log.Debug(ex, "Could not query parent process information"); + return null; + } + } + } +} + +#endif diff --git a/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs b/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs new file mode 100644 index 00000000..e688faac --- /dev/null +++ b/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs @@ -0,0 +1,57 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.IO; +using System.Text; +using Microsoft.AspNetCore.Mvc; +using Seq.Forwarder.Config; +using Seq.Forwarder.Diagnostics; +using Serilog.Formatting.Display; + +namespace Seq.Forwarder.Web.Api +{ + public class ApiRootController : Controller + { + static readonly Encoding Encoding = new UTF8Encoding(false); + readonly MessageTemplateTextFormatter _ingestionLogFormatter; + + public ApiRootController(SeqForwarderDiagnosticConfig diagnosticConfig) + { + var template = "[{Timestamp:o} {Level:u3}] {Message}{NewLine}"; + if (diagnosticConfig.IngestionLogShowDetail) + template += "Client IP address: {ClientHostIP}{NewLine}First {StartToLog} characters of payload: {DocumentStart:l}{NewLine}{Exception}{NewLine}"; + + _ingestionLogFormatter = new MessageTemplateTextFormatter(template); + } + + [HttpGet, Route("")] + public IActionResult Index() + { + var events = IngestionLog.Read(); + using var log = new StringWriter(); + foreach (var logEvent in events) + { + _ingestionLogFormatter.Format(logEvent, log); + } + + return Content(log.ToString(), "text/plain", Encoding); + } + + [HttpGet, Route("api")] + public IActionResult Resources() + { + return Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Encoding); + } + } +} diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionController.cs b/src/SeqCli/Forwarder/Web/Api/IngestionController.cs new file mode 100644 index 00000000..76fbbc11 --- /dev/null +++ b/src/SeqCli/Forwarder/Web/Api/IngestionController.cs @@ -0,0 +1,246 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Net.Http.Headers; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Seq.Forwarder.Config; +using Seq.Forwarder.Diagnostics; +using Seq.Forwarder.Multiplexing; +using Seq.Forwarder.Schema; +using Seq.Forwarder.Shipper; + +namespace Seq.Forwarder.Web.Api +{ + public class IngestionController : Controller + { + static readonly Encoding Encoding = new UTF8Encoding(false); + const string ClefMediaType = "application/vnd.serilog.clef"; + + readonly ActiveLogBufferMap _logBufferMap; + readonly SeqForwarderOutputConfig _outputConfig; + readonly ServerResponseProxy _serverResponseProxy; + + readonly JsonSerializer _rawSerializer = JsonSerializer.Create( + new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); + + public IngestionController(ActiveLogBufferMap logBufferMap, SeqForwarderOutputConfig outputConfig, ServerResponseProxy serverResponseProxy) + { + _logBufferMap = logBufferMap; + _outputConfig = outputConfig; + _serverResponseProxy = serverResponseProxy; + } + + IPAddress ClientHostIP => Request.HttpContext.Connection.RemoteIpAddress!; + + [HttpGet, Route("api/events/describe")] + public IActionResult Resources() + { + return Content("{\"Links\":{\"Raw\":\"/api/events/raw{?clef}\"}}", "application/json", Encoding); + } + + [HttpPost, Route("api/events/raw")] + public async Task Ingest() + { + var clef = DefaultedBoolQuery("clef"); + + if (clef) + return await IngestCompactFormat(); + + var contentType = (string?) Request.Headers[HeaderNames.ContentType]; + if (contentType != null && contentType.StartsWith(ClefMediaType)) + return await IngestCompactFormat(); + + return IngestRawFormat(); + } + + IActionResult IngestRawFormat() + { + // The compact format ingestion path works with async IO. + HttpContext.Features.Get()!.AllowSynchronousIO = true; + + JObject posted; + try + { + posted = _rawSerializer.Deserialize(new JsonTextReader(new StreamReader(Request.Body))) ?? + throw new RequestProcessingException("Request body payload is JSON `null`."); + } + catch (Exception ex) + { + IngestionLog.ForClient(ClientHostIP).Debug(ex,"Rejecting payload due to invalid JSON, request body could not be parsed"); + throw new RequestProcessingException("Invalid raw event JSON, body could not be parsed."); + } + + if (!(posted.TryGetValue("events", StringComparison.Ordinal, out var eventsToken) || + posted.TryGetValue("Events", StringComparison.Ordinal, out eventsToken))) + { + IngestionLog.ForClient(ClientHostIP).Debug("Rejecting payload due to invalid JSON structure"); + throw new RequestProcessingException("Invalid raw event JSON, body must contain an 'Events' array."); + } + + if (!(eventsToken is JArray events)) + { + IngestionLog.ForClient(ClientHostIP).Debug("Rejecting payload due to invalid Events property structure"); + throw new RequestProcessingException("Invalid raw event JSON, the 'Events' property must be an array."); + } + + var encoded = EncodeRawEvents(events); + return Enqueue(encoded); + } + + async Task IngestCompactFormat() + { + var rawFormat = new List(); + var reader = new StreamReader(Request.Body); + + var line = await reader.ReadLineAsync(); + var lineNumber = 1; + + while (line != null) + { + if (!string.IsNullOrWhiteSpace(line)) + { + JObject item; + try + { + item = _rawSerializer.Deserialize(new JsonTextReader(new StringReader(line))) ?? + throw new RequestProcessingException("Request body payload is JSON `null`."); + } + catch (Exception ex) + { + IngestionLog.ForPayload(ClientHostIP, line).Debug(ex, "Rejecting CLEF payload due to invalid JSON, item could not be parsed"); + throw new RequestProcessingException($"Invalid raw event JSON, item on line {lineNumber} could not be parsed."); + } + + if (!EventSchema.FromClefFormat(lineNumber, item, out var evt, out var err)) + { + IngestionLog.ForPayload(ClientHostIP, line).Debug("Rejecting CLEF payload due to invalid event JSON structure: {NormalizationError}", err); + throw new RequestProcessingException(err); + } + + rawFormat.Add(evt); + } + + line = await reader.ReadLineAsync(); + ++lineNumber; + } + + var encoded = EncodeRawEvents(rawFormat); + return Enqueue(encoded); + } + + byte[][] EncodeRawEvents(ICollection events) + { + var encoded = new byte[events.Count][]; + var i = 0; + foreach (var e in events) + { + var s = e.ToString(Formatting.None); + var payload = Encoding.UTF8.GetBytes(s); + + if (payload.Length > (int) _outputConfig.EventBodyLimitBytes) + { + IngestionLog.ForPayload(ClientHostIP, s).Debug("An oversized event was dropped"); + + var jo = e as JObject; + // ReSharper disable SuspiciousTypeConversion.Global + var timestamp = (string?) (dynamic?) jo?.GetValue("Timestamp") ?? DateTime.UtcNow.ToString("o"); + var level = (string?) (dynamic?) jo?.GetValue("Level") ?? "Warning"; + + if (jo != null) + { + jo.Remove("Timestamp"); + jo.Remove("Level"); + } + + var startToLog = (int) Math.Min(_outputConfig.EventBodyLimitBytes / 2, 1024); + var compactPrefix = e.ToString(Formatting.None).Substring(0, startToLog); + + encoded[i] = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new + { + Timestamp = timestamp, + MessageTemplate = "Seq Forwarder received and dropped an oversized event", + Level = level, + Properties = new + { + Partial = compactPrefix, + Environment.MachineName, + _outputConfig.EventBodyLimitBytes, + PayloadBytes = payload.Length + } + })); + } + else + { + encoded[i] = payload; + } + + i++; + } + + return encoded; + } + + IActionResult Enqueue(byte[][] encodedEvents) + { + var apiKey = GetRequestApiKeyToken(); + _logBufferMap.GetLogBuffer(apiKey).Enqueue(encodedEvents); + + var response = Content(_serverResponseProxy.GetResponseText(apiKey), "application/json", Encoding); + response.StatusCode = (int)HttpStatusCode.Created; + return response; + } + + string? GetRequestApiKeyToken() + { + var apiKeyToken = Request.Headers[SeqApi.ApiKeyHeaderName].FirstOrDefault(); + + if (string.IsNullOrWhiteSpace(apiKeyToken)) + apiKeyToken = Request.Query["apiKey"]; + + var normalized = apiKeyToken?.Trim(); + if (string.IsNullOrEmpty(normalized)) + return null; + + return normalized; + } + + bool DefaultedBoolQuery(string queryParameterName) + { + var parameter = Request.Query[queryParameterName]; + if (parameter.Count != 1) + return false; + + var value = (string?) parameter; + + if (value == "" && ( + Request.QueryString.Value!.Contains($"&{queryParameterName}=") || + Request.QueryString.Value.Contains($"?{queryParameterName}="))) + { + return false; + } + + return "true".Equals(value, StringComparison.OrdinalIgnoreCase) || value == "" || value == queryParameterName; + } + } +} diff --git a/src/SeqCli/Forwarder/Web/Host/ServerService.cs b/src/SeqCli/Forwarder/Web/Host/ServerService.cs new file mode 100644 index 00000000..98d0d5dc --- /dev/null +++ b/src/SeqCli/Forwarder/Web/Host/ServerService.cs @@ -0,0 +1,67 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using Microsoft.Extensions.Hosting; +using Seq.Forwarder.Diagnostics; +using Seq.Forwarder.Multiplexing; +using Serilog; + +namespace Seq.Forwarder.Web.Host +{ + class ServerService + { + readonly ActiveLogBufferMap _logBufferMap; + readonly IHost _host; + readonly string _listenUri; + + public ServerService(ActiveLogBufferMap logBufferMap, IHost host, string listenUri) + { + _logBufferMap = logBufferMap; + _host = host; + _listenUri = listenUri; + } + + public void Start() + { + try + { + Log.Debug("Starting HTTP server..."); + + _host.Start(); + + Log.Information("Seq Forwarder listening on {ListenUri}", _listenUri); + IngestionLog.Log.Debug("Seq Forwarder is accepting events"); + + _logBufferMap.Load(); + _logBufferMap.Start(); + } + catch (Exception ex) + { + Log.Fatal(ex, "Error running the server application"); + throw; + } + } + + public void Stop() + { + Log.Debug("Seq Forwarder stopping"); + + _host.StopAsync().Wait(); + _logBufferMap.Stop(); + + Log.Information("Seq Forwarder stopped cleanly"); + } + } +} diff --git a/src/SeqCli/Forwarder/Web/Host/Startup.cs b/src/SeqCli/Forwarder/Web/Host/Startup.cs new file mode 100644 index 00000000..57379a33 --- /dev/null +++ b/src/SeqCli/Forwarder/Web/Host/Startup.cs @@ -0,0 +1,40 @@ +using System.Text; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; + +namespace Seq.Forwarder.Web.Host +{ + class Startup + { + public void ConfigureServices(IServiceCollection serviceCollection) + { + serviceCollection.AddMvc(); + } + + public void Configure(IApplicationBuilder app) + { + app.Use(async (context, next) => + { + try + { + await next(); + } + catch (RequestProcessingException rex) + { + if (context.Response.HasStarted) + throw; + + context.Response.StatusCode = (int)rex.StatusCode; + context.Response.ContentType = "text/plain; charset=UTF-8"; + await context.Response.Body.WriteAsync(Encoding.UTF8.GetBytes(rex.Message)); + await context.Response.CompleteAsync(); + } + }); + app.UseRouting(); + app.UseEndpoints(endpoints => + { + endpoints.MapControllers(); + }); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/RequestProcessingException.cs b/src/SeqCli/Forwarder/Web/RequestProcessingException.cs new file mode 100644 index 00000000..d5968643 --- /dev/null +++ b/src/SeqCli/Forwarder/Web/RequestProcessingException.cs @@ -0,0 +1,30 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Net; + +namespace Seq.Forwarder.Web +{ + class RequestProcessingException : Exception + { + public RequestProcessingException(string message, HttpStatusCode statusCode = HttpStatusCode.BadRequest) + : base(message) + { + StatusCode = statusCode; + } + + public HttpStatusCode StatusCode { get; } + } +} diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index 54e9377f..5d83f459 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -1,7 +1,7 @@ - + Exe - net8.0;net8.0-windows + net8.0 seqcli ..\..\asset\SeqCli.ico win-x64;linux-x64;linux-musl-x64;osx-x64;linux-arm64;linux-musl-arm64;osx-arm64 @@ -12,21 +12,10 @@ seqcli default enable - true - true - true + false + false + false - - - WINDOWS - - - OSX - - - LINUX - - @@ -47,7 +36,9 @@ + + diff --git a/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs new file mode 100644 index 00000000..46188948 --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Multiplexing/ActiveLogBufferMapTests.cs @@ -0,0 +1,83 @@ +using System.IO; +using System.Linq; +using Seq.Forwarder.Config; +using Seq.Forwarder.Cryptography; +using Seq.Forwarder.Multiplexing; +using Seq.Forwarder.Tests.Support; +using SeqCli.Tests.Support; +using Xunit; + +namespace Seq.Forwarder.Tests.Multiplexing +{ + public class ActiveLogBufferMapTests + { + [Fact] + public void AnEmptyMapCreatesNoFiles() + { + using var tmp = new TempFolder("Buffer"); + using var map = CreateActiveLogBufferMap(tmp); + Assert.Empty(Directory.GetFileSystemEntries(tmp.Path)); + } + + [Fact] + public void TheDefaultBufferWritesDataInTheBufferRoot() + { + using var tmp = new TempFolder("Buffer"); + using var map = CreateActiveLogBufferMap(tmp); + var entry = map.GetLogBuffer(null); + Assert.NotNull(entry); + Assert.True(File.Exists(Path.Combine(tmp.Path, "data.mdb"))); + Assert.Empty(Directory.GetDirectories(tmp.Path)); + Assert.Same(entry, map.GetLogBuffer(null)); + } + + [Fact] + public void ApiKeySpecificBuffersWriteDataToSubfolders() + { + using var tmp = new TempFolder("Buffer"); + using var map = CreateActiveLogBufferMap(tmp); + string key1 = Some.ApiKey(), key2 = Some.ApiKey(); + var entry1 = map.GetLogBuffer(key1); + var entry2 = map.GetLogBuffer(key2); + + Assert.NotNull(entry1); + Assert.NotNull(entry2); + Assert.Same(entry1, map.GetLogBuffer(key1)); + Assert.NotSame(entry1, entry2); + var subdirs = Directory.GetDirectories(tmp.Path); + Assert.Equal(2, subdirs.Length); + Assert.True(File.Exists(Path.Combine(subdirs[0], "data.mdb"))); + Assert.True(File.Exists(Path.Combine(subdirs[0], ".apikey"))); + } + + [Fact] + public void EntriesSurviveReloads() + { + var apiKey = Some.ApiKey(); + var value = Some.Bytes(100); + + using var tmp = new TempFolder("Buffer"); + using (var map = CreateActiveLogBufferMap(tmp)) + { + map.GetLogBuffer(null).Enqueue(new[] {value}); + map.GetLogBuffer(apiKey).Enqueue(new[] {value}); + } + + using (var map = CreateActiveLogBufferMap(tmp)) + { + var first = map.GetLogBuffer(null).Peek(0).Single(); + var second = map.GetLogBuffer(apiKey).Peek(0).Single(); + Assert.Equal(value, first.Value); + Assert.Equal(value, second.Value); + } + } + + static ActiveLogBufferMap CreateActiveLogBufferMap(TempFolder tmp) + { + var config = new SeqForwarderConfig(); + var map = new ActiveLogBufferMap(tmp.Path, config.Storage, config.Output, new InertLogShipperFactory(), StringDataProtector.CreatePlatformDefault()); + map.Load(); + return map; + } + } +} diff --git a/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs b/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs new file mode 100644 index 00000000..2032215e --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Schema/EventSchemaTests.cs @@ -0,0 +1,73 @@ +using System.IO; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Seq.Forwarder.Schema; +using Xunit; + +namespace Seq.Forwarder.Tests.Schema +{ + public class EventSchemaTests + { + static readonly JsonSerializer RawSerializer = JsonSerializer.Create( + new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); + + [Fact] + public void ClefNormalizationAcceptsDuplicateRenderings() + { + var payload = "{\"@t\": \"2015-05-09T12:09:08.12345Z\"," + + " \"@mt\": \"{A:000} and {A:000}\"," + + " \"@r\": [\"424\",\"424\"]}"; + + AssertCanNormalizeClef(payload); + } + + [Fact] + public void ClefNormalizationPropagatesRenderings() + { + const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12,\"@r\":[\"012\"]}"; + var evt = AssertCanNormalizeClef(payload); + Assert.Single(evt.Renderings); + } + + [Fact] + public void ClefNormalizationIgnoresMissingRenderings() + { + const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12}"; + AssertCanNormalizeClef(payload); + } + + [Fact] + public void ClefNormalizationFixesTooFewRenderings1() + { + const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12,\"@r\":[]}"; + var evt = AssertCanNormalizeClef(payload); + Assert.Null(evt.Renderings); + } + + [Fact] + public void ClefNormalizationFixesTooFewRenderings2() + { + const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000} {Q:x}!\",\"P\":12,\"@r\":[\"012\"]}"; + var evt = AssertCanNormalizeClef(payload); + Assert.Null(evt.Renderings); + } + + [Fact] + public void ClefNormalizationIgnoresTooManyRenderings() + { + const string payload = "{\"@t\":\"2018-12-02T09:05:47.256725+03:00\",\"@mt\":\"Hello {P:000}!\",\"P\":12,\"@r\":[\"012\",\"013\"]}"; + var evt = AssertCanNormalizeClef(payload); + Assert.Null(evt.Renderings); + } + + static dynamic AssertCanNormalizeClef(string payload) + { + var jo = RawSerializer.Deserialize(new JsonTextReader(new StringReader(payload)))!; + + var valid = EventSchema.FromClefFormat(1, jo, out var rawFormat, out var error); + Assert.True(valid, error); + Assert.NotNull(rawFormat); + return rawFormat!; + } + } +} \ No newline at end of file diff --git a/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs b/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs new file mode 100644 index 00000000..1ac2db7c --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Shipper/ServerResponseProxyTests.cs @@ -0,0 +1,49 @@ +using Seq.Forwarder.Multiplexing; +using Seq.Forwarder.Shipper; +using Seq.Forwarder.Tests.Support; +using SeqCli.Tests.Support; +using Xunit; + +namespace Seq.Forwarder.Tests.Shipper +{ + public class ServerResponseProxyTests + { + [Fact] + public void WhenNoResponseRecordedEmptyIsReturned() + { + var proxy = new ServerResponseProxy(); + var response = proxy.GetResponseText(Some.ApiKey()); + Assert.Equal("{}", response); + } + + [Fact] + public void WhenApiKeysDontMatchEmptyResponseReturned() + { + var proxy = new ServerResponseProxy(); + proxy.SuccessResponseReturned(Some.ApiKey(), "this is never used"); + var response = proxy.GetResponseText(Some.ApiKey()); + Assert.Equal("{}", response); + } + + [Fact] + public void WhenApiKeysMatchTheResponseIsReturned() + { + var proxy = new ServerResponseProxy(); + var apiKey = Some.ApiKey(); + var responseText = "some response"; + proxy.SuccessResponseReturned(apiKey, responseText); + var response = proxy.GetResponseText(apiKey); + Assert.Equal(responseText, response); + } + + [Fact] + public void NullApiKeysAreConsideredMatching() + { + var proxy = new ServerResponseProxy(); + var responseText = "some response"; + proxy.SuccessResponseReturned(null, responseText); + var response = proxy.GetResponseText(null); + Assert.Equal(responseText, response); + } + } +} diff --git a/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs b/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs new file mode 100644 index 00000000..e6468d00 --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs @@ -0,0 +1,151 @@ +using System.Collections.Generic; +using Seq.Forwarder.Storage; +using Seq.Forwarder.Tests.Support; +using SeqCli.Tests.Support; +using Xunit; + +namespace Seq.Forwarder.Tests.Storage +{ + public class LogBufferTests + { + const ulong DefaultBufferSize = 10 * 1024 * 1024; + + [Fact] + public void ANewLogBufferIsEmpty() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + var contents = buffer.Peek((int)DefaultBufferSize); + Assert.Empty(contents); + } + + [Fact] + public void PeekingDoesNotChangeState() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + buffer.Enqueue(new[] { Some.Bytes(140) }); + + var contents = buffer.Peek((int)DefaultBufferSize); + Assert.Single(contents); + + var remainder = buffer.Peek((int)DefaultBufferSize); + Assert.Single(remainder); + } + + [Fact] + public void EnqueuedEntriesAreDequeuedFifo() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue(new[] { a1, a2 }); + buffer.Enqueue(new[] { a3 }); + + var contents = buffer.Peek((int)DefaultBufferSize); + + Assert.Equal(3, contents.Length); + Assert.Equal(a1, contents[0].Value); + Assert.Equal(a2, contents[1].Value); + Assert.Equal(a3, contents[2].Value); + } + + [Fact] + public void EntriesOverLimitArePurgedFifo() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), 4096); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue(new[] { a1, a2, a3 }); + + var contents = buffer.Peek((int)DefaultBufferSize); + + Assert.Equal(2, contents.Length); + Assert.Equal(a2, contents[0].Value); + Assert.Equal(a3, contents[1].Value); + } + + [Fact] + public void SizeHintLimitsDequeuedEventCount() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue(new[] { a1, a2, a3 }); + + var contents = buffer.Peek(300); + + Assert.Equal(2, contents.Length); + Assert.Equal(a1, contents[0].Value); + Assert.Equal(a2, contents[1].Value); + } + + [Fact] + public void AtLeastOneEventIsAlwaysDequeued() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue(new[] { a1, a2, a3 }); + + var contents = buffer.Peek(30); + + Assert.Single(contents); + Assert.Equal(a1, contents[0].Value); + } + + [Fact] + public void GivingTheLastSeenEventKeyRemovesPrecedingEvents() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue(new[] { a1, a2, a3 }); + + var contents = buffer.Peek(420); + Assert.Equal(3, contents.Length); + + buffer.Dequeue(contents[2].Key); + + var remaining = buffer.Peek(420); + Assert.Empty(remaining); + } + + [Fact] + public void GivingTheLastSeeEventKeyLeavesSuccessiveEvents() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue(new[] { a1, a2, a3 }); + + var contents = buffer.Peek(30); + Assert.Single(contents); + + buffer.Enqueue(new [] { Some.Bytes(140) }); + + buffer.Dequeue(contents[0].Key); + + var remaining = buffer.Peek(420); + Assert.Equal(3, remaining.Length); + } + + [Fact] + public void EnumerationIsInOrder() + { + using var temp = TempFolder.ForCaller(); + using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); + byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); + buffer.Enqueue(new[] { a1, a2, a3 }); + + var contents = new List(); + buffer.Enumerate((k, v) => + { + contents.Add(v); + }); + + Assert.Equal(3, contents.Count); + Assert.Equal(new[] { a1, a2, a3 }, contents); + } + } +} diff --git a/test/SeqCli.Tests/SeqCli.Tests.csproj b/test/SeqCli.Tests/SeqCli.Tests.csproj index 8383a9bc..30d682cc 100644 --- a/test/SeqCli.Tests/SeqCli.Tests.csproj +++ b/test/SeqCli.Tests/SeqCli.Tests.csproj @@ -15,6 +15,7 @@ + diff --git a/test/SeqCli.Tests/Support/Some.cs b/test/SeqCli.Tests/Support/Some.cs index 17d1f829..4d6f5e3b 100644 --- a/test/SeqCli.Tests/Support/Some.cs +++ b/test/SeqCli.Tests/Support/Some.cs @@ -1,12 +1,17 @@ using System; using System.Linq; +using System.Security.Cryptography; using Serilog.Events; using Serilog.Parsing; namespace SeqCli.Tests.Support; +#nullable enable + static class Some { + static readonly RandomNumberGenerator Rng = RandomNumberGenerator.Create(); + public static LogEvent LogEvent() { return new LogEvent( @@ -26,4 +31,16 @@ public static string UriString() { return "http://example.com"; } + + public static byte[] Bytes(int count) + { + var bytes = new byte[count]; + Rng.GetBytes(bytes); + return bytes; + } + + public static string ApiKey() + { + return string.Join("", Bytes(8).Select(v => v.ToString("x2")).ToArray()); + } } \ No newline at end of file diff --git a/test/SeqCli.Tests/Support/TempFolder.cs b/test/SeqCli.Tests/Support/TempFolder.cs new file mode 100644 index 00000000..f7d358ef --- /dev/null +++ b/test/SeqCli.Tests/Support/TempFolder.cs @@ -0,0 +1,51 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Runtime.CompilerServices; + +#nullable enable + +namespace Seq.Forwarder.Tests.Support +{ + class TempFolder : IDisposable + { + static readonly Guid Session = Guid.NewGuid(); + + public TempFolder(string name) + { + Path = System.IO.Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), + "Seq.Forwarder.Tests", + Session.ToString("n"), + name); + + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public void Dispose() + { + try + { + if (Directory.Exists(Path)) + Directory.Delete(Path, true); + } + catch (Exception ex) + { + Debug.WriteLine(ex); + } + } + + public static TempFolder ForCaller([CallerMemberName] string? caller = null) + { + if (caller == null) throw new ArgumentNullException(nameof(caller)); + return new TempFolder(caller); + } + + public string AllocateFilename(string? ext = null) + { + return System.IO.Path.Combine(Path, Guid.NewGuid().ToString("n") + "." + (ext ?? "tmp")); + } + } +}