From 4caf14beba97abb4f9f2ac885ddad7ec99f261d4 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 1 Mar 2024 13:01:30 +1000 Subject: [PATCH 1/5] Switch to minimal API --- .../Cli/Commands/Forwarder/RunCommand.cs | 80 +++++++++---------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index 62b79989..d56cfe85 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -20,6 +20,7 @@ using System.Threading.Tasks; using Autofac; using Autofac.Extensions.DependencyInjection; +using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Hosting; using SeqCli.Cli.Features; @@ -93,54 +94,51 @@ protected override async Task Run(string[] unrecognized) try { ILifetimeScope? container = null; - using var host = new HostBuilder() - .UseSerilog() - .UseServiceProviderFactory(new AutofacServiceProviderFactory()) - .ConfigureContainer(builder => - { - builder.RegisterBuildCallback(ls => container = ls); - builder.RegisterModule(new ForwarderModule(_storagePath.BufferPath, config)); - }) - .ConfigureWebHostDefaults(web => - { - web.UseStartup(); - web.UseKestrel(options => - { - options.AddServerHeader = false; - options.AllowSynchronousIO = true; - }) - .ConfigureKestrel(options => - { - var apiListenUri = new Uri(listenUri); + var builder = WebApplication.CreateBuilder(); + builder.WebHost.UseKestrel(options => + { + options.AddServerHeader = false; + options.AllowSynchronousIO = true; + }).ConfigureKestrel((context, options) => + { + var apiListenUri = new Uri(listenUri); - var ipAddress = apiListenUri.HostNameType switch - { - UriHostNameType.Basic => IPAddress.Any, - UriHostNameType.Dns => IPAddress.Any, - UriHostNameType.IPv4 => IPAddress.Parse(apiListenUri.Host), - UriHostNameType.IPv6 => IPAddress.Parse(apiListenUri.Host), - _ => throw new NotSupportedException($"Listen URI type `{apiListenUri.HostNameType}` is not supported.") - }; + var ipAddress = apiListenUri.HostNameType switch + { + UriHostNameType.Basic => IPAddress.Any, + UriHostNameType.Dns => IPAddress.Any, + UriHostNameType.IPv4 => IPAddress.Parse(apiListenUri.Host), + UriHostNameType.IPv6 => IPAddress.Parse(apiListenUri.Host), + _ => throw new NotSupportedException($"Listen URI type `{apiListenUri.HostNameType}` is not supported.") + }; - if (apiListenUri.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase)) - { - options.Listen(ipAddress, apiListenUri.Port, listenOptions => - { + if (apiListenUri.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase)) + { + options.Listen(ipAddress, apiListenUri.Port, listenOptions => + { #if WINDOWS listenOptions.UseHttps(StoreName.My, apiListenUri.Host, location: StoreLocation.LocalMachine, allowInvalid: true); #else - listenOptions.UseHttps(); + listenOptions.UseHttps(); #endif - }); - } - else - { - options.Listen(ipAddress, apiListenUri.Port); - } - }); - }) - .Build(); + }); + } + else + { + options.Listen(ipAddress, apiListenUri.Port); + } + }); + + builder + .Host.UseSerilog() + .UseServiceProviderFactory(new AutofacServiceProviderFactory()) + .ConfigureContainer(builder => + { + builder.RegisterBuildCallback(ls => container = ls); + builder.RegisterModule(new ForwarderModule(_storagePath.BufferPath, config)); + }); + using var host = builder.Build(); if (container == null) throw new Exception("Host did not build container."); From 4c4f90db144d2e35e61b4e48dc392397eb99316e Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 1 Mar 2024 13:05:42 +1000 Subject: [PATCH 2/5] Fix build --- src/SeqCli/Cli/Features/StoragePathFeature.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/SeqCli/Cli/Features/StoragePathFeature.cs b/src/SeqCli/Cli/Features/StoragePathFeature.cs index 50523ca8..4283ddea 100644 --- a/src/SeqCli/Cli/Features/StoragePathFeature.cs +++ b/src/SeqCli/Cli/Features/StoragePathFeature.cs @@ -1,6 +1,5 @@ using System; using System.IO; -using SeqCli.Forwarder.ServiceProcess; namespace SeqCli.Cli.Features; From ef46a5fb9f1bc74895bb5e89b3e8700365a8a793 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 1 Mar 2024 13:37:12 +1000 Subject: [PATCH 3/5] ApiRoot endpoints --- .../Cli/Commands/Forwarder/RunCommand.cs | 9 +++- src/SeqCli/Forwarder/ForwarderModule.cs | 7 +++ .../Forwarder/Web/Api/ApiRootController.cs | 45 ++++++++----------- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index b310c845..5b5abd4f 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -28,11 +28,13 @@ using SeqCli.Config.Forwarder; using SeqCli.Forwarder; using SeqCli.Forwarder.Util; +using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; using Serilog; using Serilog.Core; using Serilog.Events; using Serilog.Formatting.Compact; +using Serilog.Formatting.Display; #if WINDOWS using SeqCli.Forwarder.ServiceProcess; @@ -142,10 +144,13 @@ protected override async Task Run(string[] unrecognized) builder.RegisterBuildCallback(ls => container = ls); builder.RegisterModule(new ForwarderModule(_storagePath.BufferPath, config)); }); + using var host = builder.Build(); - + if (container == null) throw new Exception("Host did not build container."); - + + ApiRoot.Map(host, container.Resolve()); + var service = container.Resolve( new TypedParameter(typeof(IHost), host), new NamedParameter("listenUri", listenUri)); diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 019afd4f..8dbc69b2 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -19,7 +19,9 @@ using SeqCli.Config; using SeqCli.Encryptor; using SeqCli.Forwarder.Multiplexing; +using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; +using Serilog.Formatting.Display; namespace SeqCli.Forwarder; @@ -43,6 +45,11 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType().As(); builder.RegisterType().SingleInstance(); + builder.RegisterType(); + builder.RegisterInstance(new MessageTemplateTextFormatter( + "[{Timestamp:o} {Level:u3}] {Message}{NewLine}" + (_config.Forwarder.Diagnostics.IngestionLogShowDetail + ? "" + : "Client IP address: {ClientHostIP}{NewLine}First {StartToLog} characters of payload: {DocumentStart:l}{NewLine}{Exception}{NewLine}"))).SingleInstance(); builder.Register(c => { diff --git a/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs b/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs index 9747a8d2..3d6a2d70 100644 --- a/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs +++ b/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs @@ -14,43 +14,36 @@ using System.IO; using System.Text; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Hosting; using SeqCli.Config.Forwarder; using SeqCli.Forwarder.Diagnostics; using Serilog.Formatting.Display; namespace SeqCli.Forwarder.Web.Api; -public class ApiRootController : Controller -{ +public class ApiRoot +{ static readonly Encoding Encoding = new UTF8Encoding(false); - readonly MessageTemplateTextFormatter _ingestionLogFormatter; - public ApiRootController(ForwarderDiagnosticConfig diagnosticConfig) + public static void Map(WebApplication app, MessageTemplateTextFormatter formatter) { - var template = "[{Timestamp:o} {Level:u3}] {Message}{NewLine}"; - if (diagnosticConfig.IngestionLogShowDetail) - template += "Client IP address: {ClientHostIP}{NewLine}First {StartToLog} characters of payload: {DocumentStart:l}{NewLine}{Exception}{NewLine}"; - - _ingestionLogFormatter = new MessageTemplateTextFormatter(template); - } - - [HttpGet, Route("")] - public IActionResult Index() - { - var events = IngestionLog.Read(); - using var log = new StringWriter(); - foreach (var logEvent in events) + app.MapGet("/", () => { - _ingestionLogFormatter.Format(logEvent, log); - } + var events = IngestionLog.Read(); + using var log = new StringWriter(); + foreach (var logEvent in events) + { + formatter.Format(logEvent, log); + } - return Content(log.ToString(), "text/plain", Encoding); - } + return Results.Content(log.ToString(), "text/plain", Encoding); + }); - [HttpGet, Route("api")] - public IActionResult Resources() - { - return Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Encoding); - } + app.MapGet("/api", + () => Results.Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Encoding)); + + } } \ No newline at end of file From cea428d1d87f89cbc07a3b08e1ba58c52c348567 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 1 Mar 2024 15:13:08 +1000 Subject: [PATCH 4/5] Remove MVC --- .../Cli/Commands/Forwarder/RunCommand.cs | 9 +- src/SeqCli/Forwarder/ForwarderModule.cs | 5 +- ...iRootController.cs => ApiRootEndpoints.cs} | 23 +- src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs | 8 + ...ionController.cs => IngestionEndpoints.cs} | 249 +++++++++--------- src/SeqCli/Forwarder/Web/Host/Startup.cs | 1 - 6 files changed, 156 insertions(+), 139 deletions(-) rename src/SeqCli/Forwarder/Web/Api/{ApiRootController.cs => ApiRootEndpoints.cs} (74%) create mode 100644 src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs rename src/SeqCli/Forwarder/Web/Api/{IngestionController.cs => IngestionEndpoints.cs} (57%) diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index 5b5abd4f..4c4cfa00 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -13,6 +13,8 @@ // limitations under the License. using System; +using System.Collections; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net; @@ -148,8 +150,11 @@ protected override async Task Run(string[] unrecognized) using var host = builder.Build(); if (container == null) throw new Exception("Host did not build container."); - - ApiRoot.Map(host, container.Resolve()); + + foreach (var mapper in container.Resolve>()) + { + mapper.Map(host); + } var service = container.Resolve( new TypedParameter(typeof(IHost), host), diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 8dbc69b2..a0df187d 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -17,7 +17,6 @@ using System.Threading; using Autofac; using SeqCli.Config; -using SeqCli.Encryptor; using SeqCli.Forwarder.Multiplexing; using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; @@ -45,7 +44,9 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType().As(); builder.RegisterType().SingleInstance(); - builder.RegisterType(); + builder.RegisterType().As(); + builder.RegisterType().As(); + builder.RegisterInstance(_config.Connection); builder.RegisterInstance(new MessageTemplateTextFormatter( "[{Timestamp:o} {Level:u3}] {Message}{NewLine}" + (_config.Forwarder.Diagnostics.IngestionLogShowDetail ? "" diff --git a/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs b/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs similarity index 74% rename from src/SeqCli/Forwarder/Web/Api/ApiRootController.cs rename to src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs index 3d6a2d70..90015f1b 100644 --- a/src/SeqCli/Forwarder/Web/Api/ApiRootController.cs +++ b/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs @@ -16,19 +16,22 @@ using System.Text; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Hosting; -using SeqCli.Config.Forwarder; using SeqCli.Forwarder.Diagnostics; using Serilog.Formatting.Display; namespace SeqCli.Forwarder.Web.Api; -public class ApiRoot -{ - static readonly Encoding Encoding = new UTF8Encoding(false); +class ApiRootEndpoints : IMapEndpoints +{ + readonly MessageTemplateTextFormatter _formatter; + readonly Encoding Utf8 = new UTF8Encoding(false); - public static void Map(WebApplication app, MessageTemplateTextFormatter formatter) + public ApiRootEndpoints(MessageTemplateTextFormatter formatter) + { + _formatter = formatter; + } + + public void Map(WebApplication app) { app.MapGet("/", () => { @@ -36,14 +39,14 @@ public static void Map(WebApplication app, MessageTemplateTextFormatter formatte using var log = new StringWriter(); foreach (var logEvent in events) { - formatter.Format(logEvent, log); + _formatter.Format(logEvent, log); } - return Results.Content(log.ToString(), "text/plain", Encoding); + return Results.Content(log.ToString(), "text/plain", Utf8); }); app.MapGet("/api", - () => Results.Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Encoding)); + () => Results.Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Utf8)); } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs new file mode 100644 index 00000000..b5c53b1b --- /dev/null +++ b/src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs @@ -0,0 +1,8 @@ +using Microsoft.AspNetCore.Builder; + +namespace SeqCli.Forwarder.Web.Api; + +interface IMapEndpoints +{ + void Map(WebApplication app); +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionController.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs similarity index 57% rename from src/SeqCli/Forwarder/Web/Api/IngestionController.cs rename to src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index 6510df77..3a463c9d 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionController.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -19,8 +19,10 @@ using System.Net; using System.Text; using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.Features; -using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.Net.Http.Headers; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -32,86 +34,157 @@ namespace SeqCli.Forwarder.Web.Api; -class IngestionController : Controller +class IngestionEndpoints : IMapEndpoints { - static readonly Encoding Encoding = new UTF8Encoding(false); - const string ClefMediaType = "application/vnd.serilog.clef"; + static readonly Encoding Utf8 = new UTF8Encoding(false); readonly ActiveLogBufferMap _logBufferMap; - readonly ConnectionConfig _outputConfig; + readonly ConnectionConfig _connectionConfig; readonly ServerResponseProxy _serverResponseProxy; readonly JsonSerializer _rawSerializer = JsonSerializer.Create( new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); - public IngestionController(ActiveLogBufferMap logBufferMap, ConnectionConfig outputConfig, ServerResponseProxy serverResponseProxy) + public IngestionEndpoints( + ActiveLogBufferMap logBufferMap, + ServerResponseProxy serverResponseProxy, + ConnectionConfig connectionConfig) { _logBufferMap = logBufferMap; - _outputConfig = outputConfig; + _connectionConfig = connectionConfig; _serverResponseProxy = serverResponseProxy; } + + public void Map(WebApplication app) + { + app.MapGet("/api", + () => Results.Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Utf8)); - IPAddress ClientHostIP => Request.HttpContext.Connection.RemoteIpAddress!; + app.MapPost("api/events/raw", new Func>(async (context) => + { + var clef = DefaultedBoolQuery(context.Request, "clef"); - [HttpGet, Route("api/events/describe")] - public IActionResult Resources() - { - return Content("{\"Links\":{\"Raw\":\"/api/events/raw{?clef}\"}}", "application/json", Encoding); - } + if (clef) + return await IngestCompactFormat(context); - [HttpPost, Route("api/events/raw")] - public async Task Ingest() + var contentType = (string?) context.Request.Headers[HeaderNames.ContentType]; + var clefMediaType = "application/vnd.serilog.clef"; + + if (contentType != null && contentType.StartsWith(clefMediaType)) + return await IngestCompactFormat(context); + + return IngestRawFormat(context); + })); + } + + byte[][] EncodeRawEvents(ICollection events, IPAddress remoteIpAddress) { - var clef = DefaultedBoolQuery("clef"); + var encoded = new byte[events.Count][]; + var i = 0; + foreach (var e in events) + { + var s = e.ToString(Formatting.None); + var payload = Utf8.GetBytes(s); - if (clef) - return await IngestCompactFormat(); + if (payload.Length > (int) _connectionConfig.EventBodyLimitBytes) + { + IngestionLog.ForPayload(remoteIpAddress, s).Debug("An oversized event was dropped"); + + var jo = e as JObject; + // ReSharper disable SuspiciousTypeConversion.Global + var timestamp = (string?) (dynamic?) jo?.GetValue("Timestamp") ?? DateTime.UtcNow.ToString("o"); + var level = (string?) (dynamic?) jo?.GetValue("Level") ?? "Warning"; + + if (jo != null) + { + jo.Remove("Timestamp"); + jo.Remove("Level"); + } + + var startToLog = (int) Math.Min(_connectionConfig.EventBodyLimitBytes / 2, 1024); + var compactPrefix = e.ToString(Formatting.None).Substring(0, startToLog); + + encoded[i] = Utf8.GetBytes(JsonConvert.SerializeObject(new + { + Timestamp = timestamp, + MessageTemplate = "Seq Forwarder received and dropped an oversized event", + Level = level, + Properties = new + { + Partial = compactPrefix, + Environment.MachineName, + _connectionConfig.EventBodyLimitBytes, + PayloadBytes = payload.Length + } + })); + } + else + { + encoded[i] = payload; + } - var contentType = (string?) Request.Headers[HeaderNames.ContentType]; - if (contentType != null && contentType.StartsWith(ClefMediaType)) - return await IngestCompactFormat(); + i++; + } - return IngestRawFormat(); + return encoded; } + + static bool DefaultedBoolQuery(HttpRequest request, string queryParameterName) + { + var parameter = request.Query[queryParameterName]; + if (parameter.Count != 1) + return false; + + var value = (string?) parameter; - IActionResult IngestRawFormat() + if (value == "" && ( + request.QueryString.Value!.Contains($"&{queryParameterName}=") || + request.QueryString.Value.Contains($"?{queryParameterName}="))) + { + return false; + } + + return "true".Equals(value, StringComparison.OrdinalIgnoreCase) || value == "" || value == queryParameterName; + } + + IResult IngestRawFormat(HttpContext context) { // The compact format ingestion path works with async IO. - HttpContext.Features.Get()!.AllowSynchronousIO = true; - + context.Features.Get()!.AllowSynchronousIO = true; + JObject posted; try { - posted = _rawSerializer.Deserialize(new JsonTextReader(new StreamReader(Request.Body))) ?? + posted = _rawSerializer.Deserialize(new JsonTextReader(new StreamReader(context.Request.Body))) ?? throw new RequestProcessingException("Request body payload is JSON `null`."); } catch (Exception ex) { - IngestionLog.ForClient(ClientHostIP).Debug(ex,"Rejecting payload due to invalid JSON, request body could not be parsed"); + IngestionLog.ForClient(context.Connection.RemoteIpAddress!).Debug(ex,"Rejecting payload due to invalid JSON, request body could not be parsed"); throw new RequestProcessingException("Invalid raw event JSON, body could not be parsed."); } if (!(posted.TryGetValue("events", StringComparison.Ordinal, out var eventsToken) || posted.TryGetValue("Events", StringComparison.Ordinal, out eventsToken))) { - IngestionLog.ForClient(ClientHostIP).Debug("Rejecting payload due to invalid JSON structure"); + IngestionLog.ForClient(context.Connection.RemoteIpAddress!).Debug("Rejecting payload due to invalid JSON structure"); throw new RequestProcessingException("Invalid raw event JSON, body must contain an 'Events' array."); } if (!(eventsToken is JArray events)) { - IngestionLog.ForClient(ClientHostIP).Debug("Rejecting payload due to invalid Events property structure"); + IngestionLog.ForClient(context.Connection.RemoteIpAddress!).Debug("Rejecting payload due to invalid Events property structure"); throw new RequestProcessingException("Invalid raw event JSON, the 'Events' property must be an array."); } - var encoded = EncodeRawEvents(events); - return Enqueue(encoded); + var encoded = EncodeRawEvents(events, context.Connection.RemoteIpAddress!); + return Enqueue(context.Request, encoded); } - - async Task IngestCompactFormat() + + async Task IngestCompactFormat(HttpContext context) { var rawFormat = new List(); - var reader = new StreamReader(Request.Body); + var reader = new StreamReader(context.Request.Body); var line = await reader.ReadLineAsync(); var lineNumber = 1; @@ -128,13 +201,13 @@ async Task IngestCompactFormat() } catch (Exception ex) { - IngestionLog.ForPayload(ClientHostIP, line).Debug(ex, "Rejecting CLEF payload due to invalid JSON, item could not be parsed"); + IngestionLog.ForPayload(context.Connection.RemoteIpAddress!, line).Debug(ex, "Rejecting CLEF payload due to invalid JSON, item could not be parsed"); throw new RequestProcessingException($"Invalid raw event JSON, item on line {lineNumber} could not be parsed."); } if (!EventSchema.FromClefFormat(lineNumber, item, out var evt, out var err)) { - IngestionLog.ForPayload(ClientHostIP, line).Debug("Rejecting CLEF payload due to invalid event JSON structure: {NormalizationError}", err); + IngestionLog.ForPayload(context.Connection.RemoteIpAddress!, line).Debug("Rejecting CLEF payload due to invalid event JSON structure: {NormalizationError}", err); throw new RequestProcessingException(err); } @@ -145,101 +218,29 @@ async Task IngestCompactFormat() ++lineNumber; } - var encoded = EncodeRawEvents(rawFormat); - return Enqueue(encoded); + return Enqueue( + context.Request, + EncodeRawEvents(rawFormat, context.Connection.RemoteIpAddress!)); } - - byte[][] EncodeRawEvents(ICollection events) + + ContentHttpResult Enqueue(HttpRequest request, byte[][] encodedEvents) { - var encoded = new byte[events.Count][]; - var i = 0; - foreach (var e in events) - { - var s = e.ToString(Formatting.None); - var payload = Encoding.UTF8.GetBytes(s); - - if (payload.Length > (int) _outputConfig.EventBodyLimitBytes) - { - IngestionLog.ForPayload(ClientHostIP, s).Debug("An oversized event was dropped"); - - var jo = e as JObject; - // ReSharper disable SuspiciousTypeConversion.Global - var timestamp = (string?) (dynamic?) jo?.GetValue("Timestamp") ?? DateTime.UtcNow.ToString("o"); - var level = (string?) (dynamic?) jo?.GetValue("Level") ?? "Warning"; + var apiKeyToken = request.Headers[SeqApi.ApiKeyHeaderName].FirstOrDefault(); - if (jo != null) - { - jo.Remove("Timestamp"); - jo.Remove("Level"); - } - - var startToLog = (int) Math.Min(_outputConfig.EventBodyLimitBytes / 2, 1024); - var compactPrefix = e.ToString(Formatting.None).Substring(0, startToLog); - - encoded[i] = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new - { - Timestamp = timestamp, - MessageTemplate = "Seq Forwarder received and dropped an oversized event", - Level = level, - Properties = new - { - Partial = compactPrefix, - Environment.MachineName, - _outputConfig.EventBodyLimitBytes, - PayloadBytes = payload.Length - } - })); - } - else - { - encoded[i] = payload; - } - - i++; - } + if (string.IsNullOrWhiteSpace(apiKeyToken)) + apiKeyToken = request.Query["apiKey"]; - return encoded; - } + var apiKey = string.IsNullOrWhiteSpace(apiKeyToken) + ? null + : apiKeyToken.Trim(); - IActionResult Enqueue(byte[][] encodedEvents) - { - var apiKey = GetRequestApiKeyToken(); _logBufferMap.GetLogBuffer(apiKey).Enqueue(encodedEvents); - - var response = Content(_serverResponseProxy.GetResponseText(apiKey), "application/json", Encoding); - response.StatusCode = (int)HttpStatusCode.Created; - return response; - } - - string? GetRequestApiKeyToken() - { - var apiKeyToken = Request.Headers[SeqApi.ApiKeyHeaderName].FirstOrDefault(); - - if (string.IsNullOrWhiteSpace(apiKeyToken)) - apiKeyToken = Request.Query["apiKey"]; - - var normalized = apiKeyToken?.Trim(); - if (string.IsNullOrEmpty(normalized)) - return null; - return normalized; - } + return TypedResults.Content( + _serverResponseProxy.GetResponseText(apiKey), + "application/json", + Utf8, + StatusCodes.Status201Created); - bool DefaultedBoolQuery(string queryParameterName) - { - var parameter = Request.Query[queryParameterName]; - if (parameter.Count != 1) - return false; - - var value = (string?) parameter; - - if (value == "" && ( - Request.QueryString.Value!.Contains($"&{queryParameterName}=") || - Request.QueryString.Value.Contains($"?{queryParameterName}="))) - { - return false; - } - - return "true".Equals(value, StringComparison.OrdinalIgnoreCase) || value == "" || value == queryParameterName; } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Host/Startup.cs b/src/SeqCli/Forwarder/Web/Host/Startup.cs index 804e2065..3ca712a6 100644 --- a/src/SeqCli/Forwarder/Web/Host/Startup.cs +++ b/src/SeqCli/Forwarder/Web/Host/Startup.cs @@ -8,7 +8,6 @@ class Startup { public void ConfigureServices(IServiceCollection serviceCollection) { - serviceCollection.AddMvc(); } public void Configure(IApplicationBuilder app) From a942aeb9903ccb8362ca6ea54eb62f47c3210a9b Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 1 Mar 2024 15:15:57 +1000 Subject: [PATCH 5/5] Improve registration --- src/SeqCli/Forwarder/ForwarderModule.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index a0df187d..1bccec25 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -46,7 +46,7 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType().SingleInstance(); builder.RegisterType().As(); builder.RegisterType().As(); - builder.RegisterInstance(_config.Connection); + builder.Register(c => _config.Connection); builder.RegisterInstance(new MessageTemplateTextFormatter( "[{Timestamp:o} {Level:u3}] {Message}{NewLine}" + (_config.Forwarder.Diagnostics.IngestionLogShowDetail ? ""