From 18673f4e7b006062fa2fd90ecf1b57dbeb505cbb Mon Sep 17 00:00:00 2001 From: KodrAus Date: Thu, 9 May 2024 08:47:50 +1000 Subject: [PATCH 1/7] fix up build on Windows --- src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs | 3 --- src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs | 1 + src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs | 1 - src/SeqCli/Cli/Features/StoragePathFeature.cs | 6 +++++- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs index b3bb1c3d..c6bbfb4c 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs @@ -23,13 +23,10 @@ using System.Threading.Tasks; using SeqCli.Forwarder.Cli.Features; using SeqCli.Forwarder.Util; -using SeqCli; using SeqCli.Cli; using SeqCli.Cli.Features; using SeqCli.Config; -using SeqCli.Config.Forwarder; using SeqCli.Forwarder.ServiceProcess; -using SeqCli.Forwarder.Util; // ReSharper disable once ClassNeverInstantiated.Global diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index 4e69cebd..fb9af0a3 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -38,6 +38,7 @@ using Serilog.Formatting.Compact; #if WINDOWS +using System.Security.Cryptography.X509Certificates; using SeqCli.Forwarder.ServiceProcess; #endif diff --git a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs index 20bfbcdd..50845d26 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs @@ -20,7 +20,6 @@ using SeqCli.Forwarder.Util; using SeqCli.Cli; using SeqCli.Forwarder.ServiceProcess; -using SeqCli.Forwarder.Util; namespace SeqCli.Forwarder.Cli.Commands { diff --git a/src/SeqCli/Cli/Features/StoragePathFeature.cs b/src/SeqCli/Cli/Features/StoragePathFeature.cs index 809a62b6..b129f796 100644 --- a/src/SeqCli/Cli/Features/StoragePathFeature.cs +++ b/src/SeqCli/Cli/Features/StoragePathFeature.cs @@ -1,6 +1,10 @@ using System; using System.IO; +#if WINDOWS +using SeqCli.Forwarder.ServiceProcess; +#endif + namespace SeqCli.Cli.Features; class StoragePathFeature : CommandFeature @@ -48,7 +52,7 @@ static string GetDefaultStorageRoot() static string? TryQueryInstalledStorageRoot() { #if WINDOWS - if (SeqCli.Forwarder.Util.ServiceConfiguration.GetServiceStoragePath( + if (Forwarder.Util.ServiceConfiguration.GetServiceStoragePath( SeqCliForwarderWindowsService.WindowsServiceName, out var storage)) return storage; #endif From dc27108d9bf0ea42a094a6fdfa52956560653043 Mon Sep 17 00:00:00 2001 From: KodrAus Date: Thu, 9 May 2024 09:03:14 +1000 Subject: [PATCH 2/7] add a durable buffer implementation for forwarder --- .../LogBuffer.cs => Channel/LogChannel.cs} | 12 +- .../Forwarder/Channel/LogChannelEntry.cs | 6 + .../LogChannelMap.cs} | 10 +- .../Filesystem/EmptyStoreFileReader.cs | 29 ++ .../Forwarder/Filesystem/StoreDirectory.cs | 91 +++++ src/SeqCli/Forwarder/Filesystem/StoreFile.cs | 65 ++++ .../Forwarder/Filesystem/StoreFileAppender.cs | 37 ++ .../Forwarder/Filesystem/StoreFileReader.cs | 27 ++ .../Filesystem/System/SystemStoreDirectory.cs | 125 +++++++ .../Filesystem/System/SystemStoreFile.cs | 122 +++++++ .../System/SystemStoreFileAppender.cs | 58 ++++ .../System/SystemStoreFileReader.cs | 52 +++ .../Forwarder/Filesystem/System/Unix/Libc.cs | 29 ++ src/SeqCli/Forwarder/ForwarderModule.cs | 4 +- src/SeqCli/Forwarder/Storage/Bookmark.cs | 144 ++++++++ src/SeqCli/Forwarder/Storage/BookmarkName.cs | 56 +++ src/SeqCli/Forwarder/Storage/BookmarkValue.cs | 54 +++ .../Forwarder/Storage/BufferAppender.cs | 164 +++++++++ .../Forwarder/Storage/BufferAppenderChunk.cs | 28 ++ src/SeqCli/Forwarder/Storage/BufferReader.cs | 325 ++++++++++++++++++ .../Forwarder/Storage/BufferReaderBatch.cs | 50 +++ .../Forwarder/Storage/BufferReaderChunk.cs | 56 +++ .../Storage/BufferReaderChunkHead.cs | 23 ++ .../Forwarder/Storage/BufferReaderHead.cs | 20 ++ src/SeqCli/Forwarder/Storage/ChunkName.cs | 56 +++ src/SeqCli/Forwarder/Storage/Identifier.cs | 60 ++++ .../Forwarder/Storage/LogBufferEntry.cs | 6 - .../Forwarder/Web/Api/IngestionEndpoints.cs | 12 +- .../Forwarder/Web/Host/ServerService.cs | 10 +- src/SeqCli/SeqCli.csproj | 1 + .../Filesystem/InMemoryStoreDirectory.cs | 50 +++ .../Forwarder/Filesystem/InMemoryStoreFile.cs | 38 ++ .../Filesystem/InMemoryStoreFileAppender.cs | 39 +++ .../Filesystem/InMemoryStoreFileReader.cs | 29 ++ .../Forwarder/Storage/BookmarkTests.cs | 83 +++++ .../Forwarder/Storage/BufferTests.cs | 283 +++++++++++++++ .../Forwarder/Storage/IdentifierTests.cs | 31 ++ test/SeqCli.Tests/SeqCli.Tests.csproj | 1 + 38 files changed, 2256 insertions(+), 30 deletions(-) rename src/SeqCli/Forwarder/{Storage/LogBuffer.cs => Channel/LogChannel.cs} (78%) create mode 100644 src/SeqCli/Forwarder/Channel/LogChannelEntry.cs rename src/SeqCli/Forwarder/{Storage/LogBufferMap.cs => Channel/LogChannelMap.cs} (50%) create mode 100644 src/SeqCli/Forwarder/Filesystem/EmptyStoreFileReader.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/StoreFile.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/StoreFileAppender.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/StoreFileReader.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/System/SystemStoreFile.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileAppender.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileReader.cs create mode 100644 src/SeqCli/Forwarder/Filesystem/System/Unix/Libc.cs create mode 100644 src/SeqCli/Forwarder/Storage/Bookmark.cs create mode 100644 src/SeqCli/Forwarder/Storage/BookmarkName.cs create mode 100644 src/SeqCli/Forwarder/Storage/BookmarkValue.cs create mode 100644 src/SeqCli/Forwarder/Storage/BufferAppender.cs create mode 100644 src/SeqCli/Forwarder/Storage/BufferAppenderChunk.cs create mode 100644 src/SeqCli/Forwarder/Storage/BufferReader.cs create mode 100644 src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs create mode 100644 src/SeqCli/Forwarder/Storage/BufferReaderChunk.cs create mode 100644 src/SeqCli/Forwarder/Storage/BufferReaderChunkHead.cs create mode 100644 src/SeqCli/Forwarder/Storage/BufferReaderHead.cs create mode 100644 src/SeqCli/Forwarder/Storage/ChunkName.cs create mode 100644 src/SeqCli/Forwarder/Storage/Identifier.cs delete mode 100644 src/SeqCli/Forwarder/Storage/LogBufferEntry.cs create mode 100644 test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreDirectory.cs create mode 100644 test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs create mode 100644 test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileAppender.cs create mode 100644 test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileReader.cs create mode 100644 test/SeqCli.Tests/Forwarder/Storage/BookmarkTests.cs create mode 100644 test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs create mode 100644 test/SeqCli.Tests/Forwarder/Storage/IdentifierTests.cs diff --git a/src/SeqCli/Forwarder/Storage/LogBuffer.cs b/src/SeqCli/Forwarder/Channel/LogChannel.cs similarity index 78% rename from src/SeqCli/Forwarder/Storage/LogBuffer.cs rename to src/SeqCli/Forwarder/Channel/LogChannel.cs index 25bde0d9..4a02585b 100644 --- a/src/SeqCli/Forwarder/Storage/LogBuffer.cs +++ b/src/SeqCli/Forwarder/Channel/LogChannel.cs @@ -3,13 +3,13 @@ using System.Threading.Channels; using System.Threading.Tasks; -namespace SeqCli.Forwarder.Storage; +namespace SeqCli.Forwarder.Channel; -class LogBuffer +class LogChannel { - public LogBuffer(Func write, CancellationToken cancellationToken) + public LogChannel(Func write, CancellationToken cancellationToken) { - var channel = Channel.CreateBounded(new BoundedChannelOptions(5) + var channel = System.Threading.Channels.Channel.CreateBounded(new BoundedChannelOptions(5) { SingleReader = false, SingleWriter = true, @@ -35,7 +35,7 @@ public LogBuffer(Func write, CancellationToken cancella }, cancellationToken: _shutdownTokenSource.Token); } - readonly ChannelWriter _writer; + readonly ChannelWriter _writer; readonly Task _worker; readonly CancellationTokenSource _shutdownTokenSource; @@ -44,7 +44,7 @@ public async Task WriteAsync(byte[] storage, Range range, CancellationToken canc var tcs = new TaskCompletionSource(); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownTokenSource.Token); - await _writer.WriteAsync(new LogBufferEntry(storage, range, tcs), cts.Token); + await _writer.WriteAsync(new LogChannelEntry(storage, range, tcs), cts.Token); await tcs.Task; } diff --git a/src/SeqCli/Forwarder/Channel/LogChannelEntry.cs b/src/SeqCli/Forwarder/Channel/LogChannelEntry.cs new file mode 100644 index 00000000..866d135a --- /dev/null +++ b/src/SeqCli/Forwarder/Channel/LogChannelEntry.cs @@ -0,0 +1,6 @@ +using System; +using System.Threading.Tasks; + +namespace SeqCli.Forwarder.Channel; + +public readonly record struct LogChannelEntry(byte[] Storage, Range Range, TaskCompletionSource Completion); diff --git a/src/SeqCli/Forwarder/Storage/LogBufferMap.cs b/src/SeqCli/Forwarder/Channel/LogChannelMap.cs similarity index 50% rename from src/SeqCli/Forwarder/Storage/LogBufferMap.cs rename to src/SeqCli/Forwarder/Channel/LogChannelMap.cs index d9601814..3bb24e90 100644 --- a/src/SeqCli/Forwarder/Storage/LogBufferMap.cs +++ b/src/SeqCli/Forwarder/Channel/LogChannelMap.cs @@ -2,18 +2,18 @@ using System.Threading.Tasks; using Serilog; -namespace SeqCli.Forwarder.Storage; +namespace SeqCli.Forwarder.Channel; -class LogBufferMap +class LogChannelMap { - public LogBufferMap() + public LogChannelMap() { } - public LogBuffer Get(string? apiKey) + public LogChannel Get(string? apiKey) { - return new LogBuffer(async (c) => await Task.Delay(TimeSpan.FromSeconds(1), c), default); + return new LogChannel(async (c) => await Task.Delay(TimeSpan.FromSeconds(1), c), default); } public Task StopAsync() diff --git a/src/SeqCli/Forwarder/Filesystem/EmptyStoreFileReader.cs b/src/SeqCli/Forwarder/Filesystem/EmptyStoreFileReader.cs new file mode 100644 index 00000000..0410e23a --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/EmptyStoreFileReader.cs @@ -0,0 +1,29 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace SeqCli.Forwarder.Filesystem; + +public sealed class EmptyStoreFileReader : StoreFileReader +{ + public override void Dispose() + { + } + + public override long CopyTo(Span buffer, long from = 0, long? length = null) + { + return 0; + } +} diff --git a/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs b/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs new file mode 100644 index 00000000..e9503162 --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs @@ -0,0 +1,91 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; + +namespace SeqCli.Forwarder.Filesystem; + +/// +/// A container of s and their names. +/// +public abstract class StoreDirectory +{ + /// + /// Create a new file with the given name, linking it into the filesystem. + /// + public abstract StoreFile Create(string name); + + public virtual (string, StoreFile) CreateTemporary() + { + var tmpName = $"rc{Guid.NewGuid():N}.tmp"; + return (tmpName, Create(tmpName)); + } + + /// + /// Delete a file with the given name, returning whether the file was deleted. + /// + public abstract bool TryDelete(string name); + + /// + /// Atomically replace the contents of one file with another, creating it if it doesn't exist and deleting the other. + /// + public abstract StoreFile Replace(string toReplace, string replaceWith); + + /// + /// Atomically replace the contents of a file. + /// + public virtual StoreFile ReplaceContents(string name, Span contents, bool sync = true) + { + var (tmpName, tmpFile) = CreateTemporary(); + + try + { + if (!tmpFile.TryOpenAppend(out var opened)) + throw new Exception("Failed to write to a temporary file that was just created."); + + using var writer = opened; + writer.Append(contents); + writer.Commit(); + + if (sync) writer.Sync(); + } + catch + { + TryDelete(tmpName); + throw; + } + + return Replace(name, tmpName); + } + + /// + /// List all files in unspecified order. + /// + public abstract IEnumerable<(string Name, StoreFile File)> List(Func predicate); + + /// + /// Try get a file by name. + /// + public virtual bool TryGet(string name, [NotNullWhen(true)] out StoreFile? file) + { + file = List(n => n == name) + .Select(p => p.File) + .FirstOrDefault(); + + return file != null; + } +} diff --git a/src/SeqCli/Forwarder/Filesystem/StoreFile.cs b/src/SeqCli/Forwarder/Filesystem/StoreFile.cs new file mode 100644 index 00000000..9e998523 --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/StoreFile.cs @@ -0,0 +1,65 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics.CodeAnalysis; + +namespace SeqCli.Forwarder.Filesystem; + +public abstract class StoreFile +{ + /// + /// Get the length of this file. + /// + /// + /// True if the length was read, false if the file is invalid. + /// + public abstract bool TryGetLength([NotNullWhen(true)] out long? length); + + /// + /// Read the complete contents of this file. + /// + /// + /// The number of bytes copied. + /// + public virtual long CopyContentsTo(Span buffer) + { + if (!TryGetLength(out var length)) throw new Exception("Failed to get the length of a file."); + + if (!TryOpenRead(length.Value, out var opened)) throw new Exception("Failed to open a reader to a file."); + + using var reader = opened; + return reader.CopyTo(buffer); + } + + /// + /// Try open a reader to the file. + /// + /// + /// True if the file was opened for reading, false if the file is invalid. + /// + public abstract bool TryOpenRead(long length, [NotNullWhen(true)] out StoreFileReader? reader); + + /// + /// Open a writer to the file. + /// + /// + /// Only a single writer to a file should be open at a given time. + /// Overlapping writers may result in data corruption. + /// + /// + /// True if the file was opened for writing, false if the file is invalid. + /// + public abstract bool TryOpenAppend([NotNullWhen(true)] out StoreFileAppender? appender); +} diff --git a/src/SeqCli/Forwarder/Filesystem/StoreFileAppender.cs b/src/SeqCli/Forwarder/Filesystem/StoreFileAppender.cs new file mode 100644 index 00000000..8e04dd91 --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/StoreFileAppender.cs @@ -0,0 +1,37 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace SeqCli.Forwarder.Filesystem; + +public abstract class StoreFileAppender : IDisposable +{ + public abstract void Dispose(); + + /// + /// Append the given data to the end of the file. + /// + public abstract void Append(Span data); + + /// + /// Commit all appended data to underlying storage. + /// + public abstract long Commit(); + + /// + /// Durably sync committed data to underlying storage. + /// + public abstract void Sync(); +} diff --git a/src/SeqCli/Forwarder/Filesystem/StoreFileReader.cs b/src/SeqCli/Forwarder/Filesystem/StoreFileReader.cs new file mode 100644 index 00000000..d98feaad --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/StoreFileReader.cs @@ -0,0 +1,27 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace SeqCli.Forwarder.Filesystem; + +public abstract class StoreFileReader : IDisposable +{ + public abstract void Dispose(); + + /// + /// Copy the complete contents of the reader to the given buffer. + /// + public abstract long CopyTo(Span buffer, long from = 0, long? length = null); +} diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs new file mode 100644 index 00000000..be27c3d4 --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs @@ -0,0 +1,125 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; +using SeqCli.Forwarder.Filesystem.System.Unix; + +namespace SeqCli.Forwarder.Filesystem.System; + +public sealed class SystemStoreDirectory : StoreDirectory +{ + readonly string _directoryPath; + + public SystemStoreDirectory(string path) + { + _directoryPath = Path.GetFullPath(path); + + if (!Directory.Exists(_directoryPath)) Directory.CreateDirectory(_directoryPath); + } + + public override SystemStoreFile Create(string name) + { + var filePath = Path.Combine(_directoryPath, name); + using var _ = File.OpenHandle(filePath, FileMode.Create, FileAccess.ReadWrite, + FileShare.ReadWrite | FileShare.Delete); + Dirsync(_directoryPath); + + return new SystemStoreFile(filePath); + } + + public override (string, StoreFile) CreateTemporary() + { + // Temporary files are still created in the same directory + // This is necessary for renames to be atomic on some filesystems + var tmpName = $"rc{Guid.NewGuid():N}.tmp"; + + var filePath = Path.Combine(_directoryPath, tmpName); + using var _ = File.OpenHandle(filePath, FileMode.CreateNew, FileAccess.ReadWrite, + FileShare.ReadWrite | FileShare.Delete, FileOptions.DeleteOnClose); + + return (tmpName, new SystemStoreFile(filePath)); + } + + public override bool TryDelete(string name) + { + var filePath = Path.Combine(_directoryPath, name); + + try + { + File.Delete(filePath); + return true; + } + catch (IOException) + { + return false; + } + } + + public override SystemStoreFile Replace(string toReplace, string replaceWith) + { + var filePath = Path.Combine(_directoryPath, toReplace); + + File.Replace(Path.Combine(_directoryPath, replaceWith), filePath, null); + + return new SystemStoreFile(filePath); + } + + public override StoreFile ReplaceContents(string name, Span contents, bool sync = true) + { + var filePath = Path.Combine(_directoryPath, name); + + using var file = File.Open(filePath, FileMode.OpenOrCreate, FileAccess.ReadWrite, + FileShare.ReadWrite | FileShare.Delete); + + // NOTE: This will be atomic if: + // 1. The incoming contents are larger or equal in size to the length of the file + // 2. The incoming contents are page sized or smaller + file.Position = 0; + file.Write(contents); + + if (sync) file.Flush(true); + + return new SystemStoreFile(filePath); + } + + public override IEnumerable<(string Name, StoreFile File)> List(Func predicate) + { + foreach (var filePath in Directory.EnumerateFiles(_directoryPath)) + { + var name = Path.GetFileName(filePath); + + if (!predicate(name)) continue; + + yield return (name, new SystemStoreFile(filePath)); + } + } + + static void Dirsync(string directoryPath) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return; + + var dir = Libc.open(directoryPath, 0); + if (dir == -1) return; + + // NOTE: directory syncing here is best-effort + // If it fails for any reason we simply carry on +#pragma warning disable CA1806 + Libc.fsync(dir); + Libc.close(dir); +#pragma warning restore CA1806 + } +} diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFile.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFile.cs new file mode 100644 index 00000000..361b6741 --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFile.cs @@ -0,0 +1,122 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.IO.MemoryMappedFiles; + +namespace SeqCli.Forwarder.Filesystem.System; + +public sealed class SystemStoreFile : StoreFile +{ + static readonly FileStreamOptions AppendOptions = new() + { + Mode = FileMode.Append, + Access = FileAccess.Write, + Share = FileShare.ReadWrite | FileShare.Delete + }; + + readonly string _filePath; + + internal SystemStoreFile(string filePath) + { + _filePath = filePath; + } + + public override bool TryGetLength([NotNullWhen(true)] out long? length) + { + try + { + length = new FileInfo(_filePath).Length; + return true; + } + catch (IOException) + { + length = null; + return false; + } + } + + public override bool TryOpenRead(long length, [NotNullWhen(true)] out StoreFileReader? reader) + { + MemoryMappedFile? disposeMmap = null; + MemoryMappedViewAccessor? disposeAccessor = null; + + // If the requested length is empty then just return a dummy reader + if (length == 0) + { + reader = new EmptyStoreFileReader(); + return true; + } + + try + { + using var file = File.OpenHandle(_filePath, FileMode.OpenOrCreate, FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete, FileOptions.SequentialScan); + + disposeMmap = MemoryMappedFile.CreateFromFile(file, null, 0, MemoryMappedFileAccess.Read, + HandleInheritability.None, + false); + disposeAccessor = disposeMmap.CreateViewAccessor(0, length, MemoryMappedFileAccess.Read); + + var mmap = disposeMmap; + var accessor = disposeAccessor; + + disposeMmap = null; + disposeAccessor = null; + + reader = new SystemStoreFileReader(mmap, accessor, length); + return true; + } + // Thrown if the file length is 0 + catch (ArgumentException) + { + reader = null; + return false; + } + // Thrown if the file is truncated while creating an accessor + catch (UnauthorizedAccessException) + { + reader = null; + return false; + } + // Thrown if the file is deleted + catch (IOException) + { + reader = null; + return false; + } + finally + { + disposeMmap?.Dispose(); + disposeAccessor?.Dispose(); + } + } + + public override bool TryOpenAppend([NotNullWhen(true)] out StoreFileAppender? appender) + { + try + { + var file = File.Open(_filePath, AppendOptions); + appender = new SystemStoreFileAppender(file); + return true; + } + catch (IOException) + { + appender = null; + return false; + } + } +} diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileAppender.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileAppender.cs new file mode 100644 index 00000000..b64b9d26 --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileAppender.cs @@ -0,0 +1,58 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; + +namespace SeqCli.Forwarder.Filesystem.System; + +public sealed class SystemStoreFileAppender : StoreFileAppender +{ + readonly FileStream _file; + long _initialLength; + long _written; + + internal SystemStoreFileAppender(FileStream file) + { + _file = file; + _initialLength = _file.Length; + _written = 0; + } + + public override void Append(Span data) + { + _written += data.Length; + _file.Write(data); + } + + public override long Commit() + { + var writeHead = _initialLength + _written; + + _initialLength = writeHead; + _written = 0; + + return writeHead; + } + + public override void Sync() + { + _file.Flush(true); + } + + public override void Dispose() + { + _file.Dispose(); + } +} diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileReader.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileReader.cs new file mode 100644 index 00000000..23e95101 --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileReader.cs @@ -0,0 +1,52 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO.MemoryMappedFiles; + +namespace SeqCli.Forwarder.Filesystem.System; + +public sealed class SystemStoreFileReader : StoreFileReader +{ + readonly MemoryMappedViewAccessor _accessor; + readonly MemoryMappedFile _file; + readonly long _length; + + internal SystemStoreFileReader(MemoryMappedFile file, MemoryMappedViewAccessor accessor, long length) + { + _file = file; + + _accessor = accessor; + _length = length; + } + + public override long CopyTo(Span buffer, long from = 0, long? length = null) + { + unsafe + { + var ptr = (byte*)_accessor.SafeMemoryMappedViewHandle.DangerousGetHandle(); + var memmap = new Span(ptr + from, (int)(length ?? _length)); + + memmap.CopyTo(buffer); + + return memmap.Length; + } + } + + public override void Dispose() + { + _accessor.Dispose(); + _file.Dispose(); + } +} diff --git a/src/SeqCli/Forwarder/Filesystem/System/Unix/Libc.cs b/src/SeqCli/Forwarder/Filesystem/System/Unix/Libc.cs new file mode 100644 index 00000000..c9edc557 --- /dev/null +++ b/src/SeqCli/Forwarder/Filesystem/System/Unix/Libc.cs @@ -0,0 +1,29 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Runtime.InteropServices; + +namespace SeqCli.Forwarder.Filesystem.System.Unix; + +static class Libc +{ + [DllImport("libc")] + public static extern int open(string path, int flags); + + [DllImport("libc")] + public static extern int close(int fd); + + [DllImport("libc")] + public static extern int fsync(int fd); +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 9f75f2dc..03d24974 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -17,7 +17,7 @@ using System.Threading; using Autofac; using SeqCli.Config; -using SeqCli.Forwarder.Storage; +using SeqCli.Forwarder.Channel; using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; using Serilog.Formatting.Display; @@ -38,7 +38,7 @@ public ForwarderModule(string bufferPath, SeqCliConfig config) protected override void Load(ContainerBuilder builder) { builder.RegisterType().SingleInstance(); - builder.RegisterType().SingleInstance(); + builder.RegisterType().SingleInstance(); builder.RegisterType().As(); builder.RegisterType().As(); diff --git a/src/SeqCli/Forwarder/Storage/Bookmark.cs b/src/SeqCli/Forwarder/Storage/Bookmark.cs new file mode 100644 index 00000000..f3ec3ea4 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/Bookmark.cs @@ -0,0 +1,144 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using SeqCli.Forwarder.Filesystem; +using Path = System.IO.Path; + +namespace SeqCli.Forwarder.Storage; + +/// +/// A durable bookmark of progress processing buffers. +/// +public sealed class Bookmark +{ + readonly StoreDirectory _storeDirectory; + + readonly object _sync = new(); + BookmarkName _name; + + BookmarkValue? _value; + + Bookmark(StoreDirectory storeDirectory, BookmarkName name, BookmarkValue? value) + { + _storeDirectory = storeDirectory; + _name = name; + _value = value; + } + + public static Bookmark Open(StoreDirectory storeDirectory) + { + var (name, value) = Read(storeDirectory); + + return new Bookmark(storeDirectory, name, value); + } + + public bool TryGet([NotNullWhen(true)] out BookmarkValue? bookmark) + { + lock (_sync) + { + if (_value != null) + { + bookmark = _value.Value; + return true; + } + + bookmark = null; + return false; + } + } + + public bool TrySet(BookmarkValue value, bool sync = true) + { + lock (_sync) + { + _value = value; + } + + try + { + Write(_storeDirectory, _name, value, sync); + return true; + } + catch (IOException) + { + _name = new BookmarkName(_name.Id + 1); + return false; + } + } + + static void Write(StoreDirectory storeDirectory, BookmarkName name, BookmarkValue value, bool fsync) + { + unsafe + { + Span bookmark = stackalloc byte[16]; + value.EncodeTo(bookmark); + + storeDirectory.ReplaceContents(name.ToString(), bookmark, fsync); + } + } + + static (BookmarkName, BookmarkValue?) Read(StoreDirectory storeDirectory) + { + // NOTE: This method shouldn't throw + var bookmarks = new List<(string, BookmarkName, StoreFile)>(); + + foreach (var (candidateFileName, candidateFile) in storeDirectory + .List(candidateFileName => Path.GetExtension(candidateFileName) is ".bookmark")) + if (BookmarkName.TryParse(candidateFileName, out var parsedBookmarkName)) + bookmarks.Add((candidateFileName, parsedBookmarkName.Value, candidateFile)); + else + // The `.bookmark` file uses an unrecognized naming convention + storeDirectory.TryDelete(candidateFileName); + + switch (bookmarks.Count) + { + // There aren't any bookmarks; return a default one + case 0: + return (new BookmarkName(0), null); + // There are old bookmark values floating around; try delete them again + case > 1: + { + bookmarks.Sort((a, b) => a.Item2.Id.CompareTo(b.Item2.Id)); + + foreach (var (toDelete, _, _) in bookmarks.Take(bookmarks.Count - 1)) + storeDirectory.TryDelete(toDelete); + break; + } + } + + var (fileName, bookmarkName, file) = bookmarks[^1]; + + try + { + unsafe + { + Span bookmark = stackalloc byte[16]; + if (file.CopyContentsTo(bookmark) != 16) throw new Exception("The bookmark is corrupted."); + + return (bookmarkName, BookmarkValue.Decode(bookmark)); + } + } + catch + { + storeDirectory.TryDelete(fileName); + + return (new BookmarkName(bookmarkName.Id + 1), new BookmarkValue()); + } + } +} diff --git a/src/SeqCli/Forwarder/Storage/BookmarkName.cs b/src/SeqCli/Forwarder/Storage/BookmarkName.cs new file mode 100644 index 00000000..76206472 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BookmarkName.cs @@ -0,0 +1,56 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Diagnostics.CodeAnalysis; + +namespace SeqCli.Forwarder.Storage; + +/// +/// A bookmark file name with its incrementing identifier. +/// +public readonly record struct BookmarkName +{ + readonly string _name; + + public readonly ulong Id; + + public BookmarkName(ulong id) + { + Id = id; + _name = Identifier.Format(id, ".bookmark"); + } + + BookmarkName(ulong id, string name) + { + Id = id; + _name = name; + } + + public static bool TryParse(string name, [NotNullWhen(true)] out BookmarkName? parsed) + { + if (Identifier.TryParse(name, ".bookmark", out var id)) + { + parsed = new BookmarkName(id.Value, name); + return true; + } + + parsed = null; + return false; + } + + public override string ToString() + { + return _name; + } +} diff --git a/src/SeqCli/Forwarder/Storage/BookmarkValue.cs b/src/SeqCli/Forwarder/Storage/BookmarkValue.cs new file mode 100644 index 00000000..72716391 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BookmarkValue.cs @@ -0,0 +1,54 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Buffers.Binary; + +namespace SeqCli.Forwarder.Storage; + +/// +/// The in-memory value of a bookmark. +/// +public readonly record struct BookmarkValue(ulong Id, long CommitHead) +{ + public void EncodeTo(Span bookmark) + { + if (bookmark.Length != 16) throw new Exception($"Bookmark values must be 16 bytes (got {bookmark.Length})."); + + BinaryPrimitives.WriteUInt64LittleEndian(bookmark, Id); + BinaryPrimitives.WriteInt64LittleEndian(bookmark[8..], CommitHead); + } + + public byte[] Encode() + { + var buffer = new byte[16]; + EncodeTo(buffer); + + return buffer; + } + + public static BookmarkValue Decode(Span bookmark) + { + if (bookmark.Length != 16) throw new Exception($"Bookmark values must be 16 bytes (got {bookmark.Length})."); + + var id = BinaryPrimitives.ReadUInt64LittleEndian(bookmark); + var commitHead = BinaryPrimitives.ReadInt64LittleEndian(bookmark[8..]); + + return new BookmarkValue + { + Id = id, + CommitHead = commitHead + }; + } +} diff --git a/src/SeqCli/Forwarder/Storage/BufferAppender.cs b/src/SeqCli/Forwarder/Storage/BufferAppender.cs new file mode 100644 index 00000000..b88576fd --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BufferAppender.cs @@ -0,0 +1,164 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using SeqCli.Forwarder.Filesystem; + +namespace SeqCli.Forwarder.Storage; + +/// +/// The write-side of a buffer. +/// +public sealed class BufferAppender : IDisposable +{ + readonly StoreDirectory _storeDirectory; + BufferAppenderChunk? _currentChunk; + + BufferAppender(StoreDirectory storeDirectory) + { + _storeDirectory = storeDirectory; + _currentChunk = null; + } + + public void Dispose() + { + _currentChunk?.Dispose(); + } + + public static BufferAppender Open(StoreDirectory storeDirectory) + { + return new BufferAppender(storeDirectory); + } + + /// + /// Try write a batch. + /// + /// This method does not throw. + /// + /// This method will write the batch into the currently active chunk file unless: + /// + /// 1. The length of the current chunk is greater than or, + /// 2. There is no current chunk, because no writes have been made, or it encountered an IO error previously. + /// + /// If either of these cases is true, then the write will be made to a new chunk file. + /// + /// The newline-delimited data to write. A batch may contain multiple values separated by + /// newlines, but must end on a newline. + /// The file size to roll on. A single batched write may cause the currently + /// active chunk to exceed this size, but a subsequent write will roll over to a new file. + /// The maximum number of chunk files to keep before starting to delete them. This + /// is an optional parameter to use in cases where the reader isn't keeping up with the writer. + /// Whether to explicitly flush the write to disk. + /// True if the write fully succeeded. If this method returns false, it is safe to retry the write, + /// but it may result in duplicate data in the case of partial success. + public bool TryAppend(Span batch, long targetChunkLength, int? maxChunks = null, bool sync = true) + { + if (batch.Length == 0) return true; + + if (batch[^1] != (byte)'\n') throw new Exception("Batches must end with a newline character (\\n)"); + + if (_currentChunk != null) + // Only use the existing chunk if it's writable and shouldn't be rolled over + if (_currentChunk.WriteHead > targetChunkLength) + { + // Run a sync before moving to a new file, just to make sure any + // buffered data makes its way to disk + _currentChunk.Appender.Sync(); + + _currentChunk.Dispose(); + _currentChunk = null; + } + + // If there's no suitable candidate chunk then create a new one + if (_currentChunk == null) + { + var nextChunkId = ReadChunks(_storeDirectory, maxChunks); + + var chunkName = new ChunkName(nextChunkId); + + var chunkFile = _storeDirectory.Create(chunkName.ToString()); + + if (chunkFile.TryOpenAppend(out var opened)) + _currentChunk = new BufferAppenderChunk(opened); + else + return false; + } + + try + { + _currentChunk.Appender.Append(batch); + _currentChunk.Appender.Commit(); + + if (sync) _currentChunk.Appender.Sync(); + + _currentChunk.WriteHead += batch.Length; + + return true; + } + catch (IOException) + { + // Don't try an explicit sync here, because the file already failed to perform IO + + _currentChunk.Dispose(); + _currentChunk = null; + + return false; + } + } + + static ulong ReadChunks(StoreDirectory storeDirectory, int? maxChunks) + { + ulong nextChunkId = 0; + + List? chunks = null; + foreach (var (fileName, _) in storeDirectory.List(candidateName => + Path.GetExtension(candidateName) is ".clef")) + { + if (!ChunkName.TryParse(fileName, out var parsedChunkName)) continue; + + nextChunkId = Math.Max(nextChunkId, parsedChunkName.Value.Id); + + if (maxChunks == null) continue; + + chunks ??= []; + chunks.Add(parsedChunkName.Value); + } + + // Apply retention on the number of chunk files if the reader isn't keeping up + if (chunks != null) + { + ApplyPreWriteRetention(storeDirectory, maxChunks!.Value, chunks); + } + + return nextChunkId + 1; + } + + static void ApplyPreWriteRetention(StoreDirectory storeDirectory, int maxChunks, List unsortedChunks) + { + // We're going to create a new buffer file, so leave room for it if a max is specified + maxChunks = Math.Max(0, maxChunks - 1); + + unsortedChunks.Sort((a, b) => a.Id.CompareTo(b.Id)); + var sortedChunks = unsortedChunks; + + if (sortedChunks.Count > maxChunks) + foreach (var delete in sortedChunks.Take(sortedChunks.Count - maxChunks)) + // This call may fail if a reader is actively holding this file open + // In these cases we let the writer proceed instead of blocking + storeDirectory.TryDelete(delete.ToString()); + } +} diff --git a/src/SeqCli/Forwarder/Storage/BufferAppenderChunk.cs b/src/SeqCli/Forwarder/Storage/BufferAppenderChunk.cs new file mode 100644 index 00000000..f68acd67 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BufferAppenderChunk.cs @@ -0,0 +1,28 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using SeqCli.Forwarder.Filesystem; + +namespace SeqCli.Forwarder.Storage; + +sealed record BufferAppenderChunk(StoreFileAppender Appender) : IDisposable +{ + public long WriteHead { get; set; } + + public void Dispose() + { + Appender.Dispose(); + } +} diff --git a/src/SeqCli/Forwarder/Storage/BufferReader.cs b/src/SeqCli/Forwarder/Storage/BufferReader.cs new file mode 100644 index 00000000..f2746b8c --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BufferReader.cs @@ -0,0 +1,325 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using SeqCli.Forwarder.Filesystem; + +namespace SeqCli.Forwarder.Storage; + +/// +/// The read-side of a buffer. +/// +public sealed class BufferReader +{ + readonly StoreDirectory _storeDirectory; + BufferReaderHead? _discardingHead; + BufferReaderHead? _readHead; + List _sortedChunks; + + BufferReader(StoreDirectory storeDirectory) + { + _sortedChunks = new List(); + _storeDirectory = storeDirectory; + _discardingHead = null; + _readHead = null; + } + + public static BufferReader Open(StoreDirectory storeDirectory) + { + var reader = new BufferReader(storeDirectory); + reader.ReadChunks(); + + return reader; + } + + /// + /// Try fill a batch from the underlying file set. + /// + /// This method does not throw. + /// + /// This method is expected to be called in a loop to continue filling and processing batches as they're written. + /// + /// Once the batch is processed, call to advance the reader past it. + /// + /// The maximum size in bytes of a batch to read. If a single value between newlines is larger + /// than this size then it will be discarded rather than read. + /// The newline-delimited batch of values read. + /// True if a batch was filled. If this method returns false, then it means there is either no new + /// data to read, some oversize data was discarded, or an IO error was encountered. + public bool TryFillBatch(int maxSize, [NotNullWhen(true)] out BufferReaderBatch? batch) + { + /* + This is where the meat of the buffer reader lives. Reading batches runs in two broad steps: + + 1. If a previous batch overflowed the buffer then we're in "discard mode". + Scan through the offending chunk until a newline delimiter is found. + 2. After discarding, attempt to fill a buffer with as much data as possible + from the underlying chunks. + */ + + if (_discardingHead != null) + { + var discardingRentedArray = ArrayPool.Shared.Rent(maxSize); + + // NOTE: We don't use `maxSize` here, because we're discarding these bytes + // so it doesn't matter what size the target array is + var discardingBatchBuffer = discardingRentedArray.AsSpan(); + + while (_discardingHead != null) + { + var chunk = _sortedChunks[0]; + + // If the chunk has changed (it may have been deleted externally) + // then stop discarding + if (chunk.Name.Id != _discardingHead.Value.Chunk) + { + _discardingHead = null; + + ArrayPool.Shared.Return(discardingRentedArray); + break; + } + + var chunkHead = Head(chunk); + + // Attempt to fill the buffer with data from the underlying chunk + if (!TryFillChunk(chunk, + chunkHead with { CommitHead = _discardingHead.Value.CommitHead }, + discardingBatchBuffer, + out var fill)) + { + // If attempting to read from the chunk fails then remove it and carry on + // This is also done below in the regular read-loop if reading fails + _sortedChunks.RemoveAt(0); + _discardingHead = null; + + ArrayPool.Shared.Return(discardingRentedArray); + break; + } + + // Scan forwards for the next newline + var firstNewlineIndex = discardingBatchBuffer[..fill.Value].IndexOf((byte)'\n'); + + // If a newline was found then advance the reader to it and stop discarding + if (firstNewlineIndex >= 0) fill = firstNewlineIndex + 1; + + _discardingHead = _discardingHead.Value with + { + CommitHead = _discardingHead.Value.CommitHead + fill.Value + }; + _readHead = _discardingHead; + + var isChunkFinished = _discardingHead.Value.CommitHead == chunkHead.WriteHead; + + // If the chunk is finished or a newline is found then stop discarding + if (firstNewlineIndex >= 0 || (isChunkFinished && _sortedChunks.Count > 1)) + { + _discardingHead = null; + + ArrayPool.Shared.Return(discardingRentedArray); + break; + } + + // If there's more data in the chunk to read then loop back through + if (!isChunkFinished) continue; + + // If the chunk is finished but a newline wasn't found then refresh + // our set of chunks and loop back through + ReadChunks(); + + ArrayPool.Shared.Return(discardingRentedArray); + batch = null; + return false; + } + } + + // Fill a buffer with newline-delimited values + + var rentedArray = ArrayPool.Shared.Rent(maxSize); + var batchBuffer = rentedArray.AsSpan()[..maxSize]; + var batchLength = 0; + + BufferReaderHead? batchHead = null; + var chunkIndex = 0; + + // Try fill the buffer with as much data as possible + // by walking over all chunks + while (chunkIndex < _sortedChunks.Count) + { + var chunk = _sortedChunks[chunkIndex]; + var chunkHead = Head(chunk); + + if (!TryFillChunk(chunk, chunkHead, batchBuffer[batchLength..], out var fill)) + { + // If we can't read from this chunk anymore then remove it and continue + _sortedChunks.RemoveAt(chunkIndex); + continue; + } + + var isBufferFull = batchLength + fill == maxSize; + var isChunkFinished = fill == chunkHead.WriteHead; + + // If either the buffer has been filled or we've reached the end of a chunk + // then scan to the last newline + if (isBufferFull || isChunkFinished) + { + // If the chunk is finished then we expect this to immediately find a trailing newline + // NOTE: `Span.LastIndexOf` and similar methods are vectorized + var lastNewlineIndex = batchBuffer[batchLength..(batchLength + fill.Value)].LastIndexOf((byte)'\n'); + if (lastNewlineIndex == -1) + { + // If this isn't the last chunk then discard the trailing data and move on + if (isChunkFinished && chunkIndex < _sortedChunks.Count) + { + chunkIndex += 1; + continue; + } + + // If this is the first chunk then we've hit an oversize payload + if (chunkIndex == 0) + { + _discardingHead = new BufferReaderHead(chunk.Name.Id, chunkHead.CommitHead + fill.Value); + + // Ensures we don't attempt to yield the data we've read + batchHead = null; + } + + // If the chunk isn't finished then the buffer is full + break; + } + + fill = lastNewlineIndex + 1; + } + + batchLength += fill.Value; + batchHead = new BufferReaderHead(chunk.Name.Id, chunkHead.CommitHead + fill.Value); + + chunkIndex += 1; + } + + // If the batch is empty (because there are no chunks or there's no new data) + // then refresh the set of chunks and return + if (batchHead == null || batchLength == 0) + { + ReadChunks(); + + ArrayPool.Shared.Return(rentedArray); + batch = null; + return false; + } + + // If the batch is non-empty then return it + batch = new BufferReaderBatch(batchHead.Value, ArrayPool.Shared, rentedArray, batchLength); + return true; + } + + /// + /// Advance the reader over a previously read batch. + /// + /// This method does not throw. + /// + /// The new head to resume reading from. + public void AdvanceTo(BufferReaderHead newReaderHead) + { + var removeLength = 0; + foreach (var chunk in _sortedChunks) + { + // A portion of the chunk is being skipped + if (chunk.Name.Id == newReaderHead.Chunk) break; + + // The remainder of the chunk is being skipped + if (chunk.Name.Id < newReaderHead.Chunk) + _storeDirectory.TryDelete(chunk.Name.ToString()); + else + throw new Exception("Chunks are out of order."); + + removeLength += 1; + } + + _readHead = newReaderHead; + _sortedChunks.RemoveRange(0, removeLength); + } + + BufferReaderChunkHead Head(BufferReaderChunk chunk) + { + if (_readHead != null && chunk.Name.Id == _readHead.Value.Chunk) + return chunk.Chunk.TryGetLength(out var writeHead) + ? new BufferReaderChunkHead(Math.Min(_readHead.Value.CommitHead, writeHead.Value), writeHead.Value) + : new BufferReaderChunkHead(_readHead.Value.CommitHead, _readHead.Value.CommitHead); + + chunk.Chunk.TryGetLength(out var length); + return new BufferReaderChunkHead(0, length ?? 0); + } + + void ReadChunks() + { + var head = _readHead ?? new BufferReaderHead(0, 0); + + List chunks = new(); + + foreach (var (fileName, file) in _storeDirectory + .List(candidateName => Path.GetExtension(candidateName) is ".clef")) + { + if (!ChunkName.TryParse(fileName, out var parsedChunkName)) continue; + + if (parsedChunkName.Value.Id >= head.Chunk) + chunks.Add(new BufferReaderChunk(parsedChunkName.Value, file)); + else + // If the chunk is before the one we're expecting to read then delete it; we've already processed it + _storeDirectory.TryDelete(fileName); + } + + chunks.Sort((a, b) => a.Name.Id.CompareTo(b.Name.Id)); + + var toDispose = _sortedChunks; + _sortedChunks = chunks; + + foreach (var chunk in toDispose) + try + { + chunk.Dispose(); + } + catch + { + // Ignored + } + } + + static bool TryFillChunk(BufferReaderChunk chunk, BufferReaderChunkHead chunkHead, Span buffer, + [NotNullWhen(true)] out int? filled) + { + var remaining = buffer.Length; + var fill = (int)Math.Min(remaining, chunkHead.Unadvanced); + + try + { + if (!chunk.TryCopyTo(buffer, chunkHead, fill)) + { + filled = null; + return false; + } + + filled = fill; + return true; + } + catch (IOException) + { + filled = null; + return false; + } + } +} diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs b/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs new file mode 100644 index 00000000..069c7535 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs @@ -0,0 +1,50 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Buffers; + +namespace SeqCli.Forwarder.Storage; + +/// +/// A contiguous batch of records pulled from a reader. +/// +public readonly record struct BufferReaderBatch +{ + readonly int _length; + + readonly ArrayPool? _pool; + readonly byte[] _storage; + + public BufferReaderBatch(BufferReaderHead readerHead, ArrayPool? pool, byte[] storage, int length) + { + ReaderHead = readerHead; + + _pool = pool; + _storage = storage; + _length = length; + } + + public BufferReaderHead ReaderHead { get; } + + public ReadOnlySpan AsSpan() + { + return _storage.AsSpan()[.._length]; + } + + public void Return() + { + _pool?.Return(_storage); + } +} diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderChunk.cs b/src/SeqCli/Forwarder/Storage/BufferReaderChunk.cs new file mode 100644 index 00000000..cf7f9b0c --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BufferReaderChunk.cs @@ -0,0 +1,56 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using SeqCli.Forwarder.Filesystem; + +namespace SeqCli.Forwarder.Storage; + +/// +/// An active chunk in a . +/// +record BufferReaderChunk(ChunkName Name, StoreFile Chunk) : IDisposable +{ + (long, StoreFileReader)? _reader; + + public void Dispose() + { + _reader?.Item2.Dispose(); + } + + public bool TryCopyTo(Span buffer, BufferReaderChunkHead head, int fill) + { + var readEnd = head.CommitHead + fill; + + if (_reader != null) + if (_reader.Value.Item1 < readEnd) + { + var toDispose = _reader.Value.Item2; + _reader = null; + + toDispose.Dispose(); + } + + if (_reader == null) + { + if (!Chunk.TryOpenRead(head.WriteHead, out var reader)) return false; + + _reader = (head.WriteHead, reader); + } + + _reader.Value.Item2.CopyTo(buffer, head.CommitHead, fill); + + return true; + } +} diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderChunkHead.cs b/src/SeqCli/Forwarder/Storage/BufferReaderChunkHead.cs new file mode 100644 index 00000000..a1778342 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BufferReaderChunkHead.cs @@ -0,0 +1,23 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace SeqCli.Forwarder.Storage; + +/// +/// The current position in a . +/// +public readonly record struct BufferReaderChunkHead(long CommitHead, long WriteHead) +{ + public long Unadvanced => WriteHead - CommitHead; +} diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderHead.cs b/src/SeqCli/Forwarder/Storage/BufferReaderHead.cs new file mode 100644 index 00000000..0240fa51 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/BufferReaderHead.cs @@ -0,0 +1,20 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace SeqCli.Forwarder.Storage; + +/// +/// A position in a . +/// +public readonly record struct BufferReaderHead(ulong Chunk, long CommitHead); diff --git a/src/SeqCli/Forwarder/Storage/ChunkName.cs b/src/SeqCli/Forwarder/Storage/ChunkName.cs new file mode 100644 index 00000000..fc301cf4 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/ChunkName.cs @@ -0,0 +1,56 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Diagnostics.CodeAnalysis; + +namespace SeqCli.Forwarder.Storage; + +/// +/// A chunk file name with its incrementing identifier. +/// +public readonly record struct ChunkName +{ + readonly string _name; + + public readonly ulong Id; + + public ChunkName(ulong id) + { + Id = id; + _name = Identifier.Format(id, ".clef"); + } + + ChunkName(ulong id, string name) + { + Id = id; + _name = name; + } + + public static bool TryParse(string name, [NotNullWhen(true)] out ChunkName? parsed) + { + if (Identifier.TryParse(name, ".clef", out var id)) + { + parsed = new ChunkName(id.Value, name); + return true; + } + + parsed = null; + return false; + } + + public override string ToString() + { + return _name; + } +} diff --git a/src/SeqCli/Forwarder/Storage/Identifier.cs b/src/SeqCli/Forwarder/Storage/Identifier.cs new file mode 100644 index 00000000..000e0f07 --- /dev/null +++ b/src/SeqCli/Forwarder/Storage/Identifier.cs @@ -0,0 +1,60 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; + +namespace SeqCli.Forwarder.Storage; + +/// +/// Utilities for parsing and formatting file names with sortable identifiers. +/// +public static class Identifier +{ + /// + /// Try parse the identifier from the given name with the given extension. + /// + public static bool TryParse(string name, string extension, [NotNullWhen(true)] out ulong? parsed) + { + if (name.Length != 16 + extension.Length) + { + parsed = null; + return false; + } + + if (!name.EndsWith(extension)) + { + parsed = null; + return false; + } + + if (ulong.TryParse(name.AsSpan()[..16], NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var id)) + { + parsed = id; + return true; + } + + parsed = null; + return false; + } + + /// + /// Format an identifier with the given identifier and extension. + /// + public static string Format(ulong id, string extension) + { + return $"{id:x16}{extension}"; + } +} diff --git a/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs b/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs deleted file mode 100644 index 4477b926..00000000 --- a/src/SeqCli/Forwarder/Storage/LogBufferEntry.cs +++ /dev/null @@ -1,6 +0,0 @@ -using System; -using System.Threading.Tasks; - -namespace SeqCli.Forwarder.Storage; - -public readonly record struct LogBufferEntry(byte[] Storage, Range Range, TaskCompletionSource Completion); diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index e511336e..e98c3828 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -28,8 +28,8 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using SeqCli.Config; +using SeqCli.Forwarder.Channel; using SeqCli.Forwarder.Diagnostics; -using SeqCli.Forwarder.Storage; using JsonException = System.Text.Json.JsonException; using JsonSerializer = Newtonsoft.Json.JsonSerializer; @@ -40,17 +40,17 @@ class IngestionEndpoints : IMapEndpoints static readonly Encoding Utf8 = new UTF8Encoding(false); readonly ConnectionConfig _connectionConfig; - readonly LogBufferMap _logBuffers; + readonly LogChannelMap _logChannels; readonly JsonSerializer _rawSerializer = JsonSerializer.Create( new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }); public IngestionEndpoints( SeqCliConfig config, - LogBufferMap logBuffers) + LogChannelMap logChannels) { _connectionConfig = config.Connection; - _logBuffers = logBuffers; + _logChannels = logChannels; } public void MapEndpoints(WebApplication app) @@ -105,7 +105,7 @@ async Task IngestCompactFormat(HttpContext context) var cts = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted); cts.CancelAfter(TimeSpan.FromSeconds(5)); - var log = _logBuffers.Get(ApiKey(context.Request)); + var log = _logChannels.Get(ApiKey(context.Request)); var payload = ArrayPool.Shared.Rent(1024 * 1024 * 10); var writeHead = 0; @@ -235,7 +235,7 @@ static bool ValidateClef(Span evt) return true; } - static async Task Write(LogBuffer log, ArrayPool pool, byte[] storage, Range range, CancellationToken cancellationToken) + static async Task Write(LogChannel log, ArrayPool pool, byte[] storage, Range range, CancellationToken cancellationToken) { try { diff --git a/src/SeqCli/Forwarder/Web/Host/ServerService.cs b/src/SeqCli/Forwarder/Web/Host/ServerService.cs index ee4b473b..6832e953 100644 --- a/src/SeqCli/Forwarder/Web/Host/ServerService.cs +++ b/src/SeqCli/Forwarder/Web/Host/ServerService.cs @@ -15,8 +15,8 @@ using System; using System.Threading.Tasks; using Microsoft.Extensions.Hosting; +using SeqCli.Forwarder.Channel; using SeqCli.Forwarder.Diagnostics; -using SeqCli.Forwarder.Storage; using Serilog; namespace SeqCli.Forwarder.Web.Host; @@ -24,13 +24,13 @@ namespace SeqCli.Forwarder.Web.Host; class ServerService { readonly IHost _host; - readonly LogBufferMap _logBufferMap; + readonly LogChannelMap _logChannelMap; readonly string _listenUri; - public ServerService(IHost host, LogBufferMap logBufferMap, string listenUri) + public ServerService(IHost host, LogChannelMap logChannelMap, string listenUri) { _host = host; - _logBufferMap = logBufferMap; + _logChannelMap = logChannelMap; _listenUri = listenUri; } @@ -60,6 +60,6 @@ public async Task StopAsync() Log.Information("HTTP server stopped; flushing buffers..."); - await _logBufferMap.StopAsync(); + await _logChannelMap.StopAsync(); } } \ No newline at end of file diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index fbb39cf1..46b724a3 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -18,6 +18,7 @@ true true true + true WINDOWS diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreDirectory.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreDirectory.cs new file mode 100644 index 00000000..7382bc1d --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreDirectory.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using SeqCli.Forwarder.Filesystem; + +namespace SeqCli.Tests.Forwarder.Filesystem; + +public class InMemoryStoreDirectory : StoreDirectory +{ + readonly Dictionary _files = new(); + + public IReadOnlyDictionary Files => _files; + + public override InMemoryStoreFile Create(string name) + { + if (_files.ContainsKey(name)) throw new Exception($"The file {name} already exists"); + + _files.Add(name, new InMemoryStoreFile()); + + return _files[name]; + } + + public InMemoryStoreFile Create(string name, Span contents) + { + var file = Create(name); + file.Append(contents); + + return file; + } + + public override bool TryDelete(string name) + { + return _files.Remove(name); + } + + public override InMemoryStoreFile Replace(string toReplace, string replaceWith) + { + _files[toReplace] = _files[replaceWith]; + _files.Remove(replaceWith); + + return _files[toReplace]; + } + + public override IEnumerable<(string Name, StoreFile File)> List(Func predicate) + { + return _files + .Where(kv => predicate(kv.Key)) + .Select(kv => (kv.Key, kv.Value as StoreFile)); + } +} diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs new file mode 100644 index 00000000..1a48b246 --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs @@ -0,0 +1,38 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using SeqCli.Forwarder.Filesystem; + +namespace SeqCli.Tests.Forwarder.Filesystem; + +public class InMemoryStoreFile : StoreFile +{ + public byte[] Contents { get; private set; } = Array.Empty(); + + public override bool TryGetLength([NotNullWhen(true)] out long? length) + { + length = Contents.Length; + return true; + } + + public void Append(Span incoming) + { + var newContents = new byte[Contents.Length + incoming.Length]; + + Contents.CopyTo(newContents.AsSpan()); + incoming.CopyTo(newContents.AsSpan()[^incoming.Length..]); + + Contents = newContents; + } + + public override bool TryOpenRead(long length, [NotNullWhen(true)] out StoreFileReader? reader) + { + reader = new InMemoryStoreFileReader(this, (int)length); + return true; + } + + public override bool TryOpenAppend([NotNullWhen(true)] out StoreFileAppender? appender) + { + appender = new InMemoryStoreFileAppender(this); + return true; + } +} diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileAppender.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileAppender.cs new file mode 100644 index 00000000..aeaf66e1 --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileAppender.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using SeqCli.Forwarder.Filesystem; + +namespace SeqCli.Tests.Forwarder.Filesystem; + +public class InMemoryStoreFileAppender : StoreFileAppender +{ + readonly List _incoming; + + readonly InMemoryStoreFile _storeFile; + + public InMemoryStoreFileAppender(InMemoryStoreFile storeFile) + { + _storeFile = storeFile; + _incoming = new List(); + } + + public override void Append(Span data) + { + _incoming.AddRange(data); + } + + public override long Commit() + { + _storeFile.Append(_incoming.ToArray()); + _incoming.Clear(); + + return _storeFile.Contents.Length; + } + + public override void Sync() + { + } + + public override void Dispose() + { + } +} diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileReader.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileReader.cs new file mode 100644 index 00000000..56bd0838 --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileReader.cs @@ -0,0 +1,29 @@ +using System; +using SeqCli.Forwarder.Filesystem; + +namespace SeqCli.Tests.Forwarder.Filesystem; + +public class InMemoryStoreFileReader : StoreFileReader +{ + readonly int _length; + + readonly InMemoryStoreFile _storeFile; + + public InMemoryStoreFileReader(InMemoryStoreFile storeFile, int length) + { + _storeFile = storeFile; + _length = length; + } + + public override long CopyTo(Span buffer, long from = 0, long? length = null) + { + var span = _storeFile.Contents.AsSpan().Slice((int)from, (int?)length ?? _length); + span.CopyTo(buffer); + + return span.Length; + } + + public override void Dispose() + { + } +} diff --git a/test/SeqCli.Tests/Forwarder/Storage/BookmarkTests.cs b/test/SeqCli.Tests/Forwarder/Storage/BookmarkTests.cs new file mode 100644 index 00000000..3d67ecbd --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Storage/BookmarkTests.cs @@ -0,0 +1,83 @@ +using System.Linq; +using SeqCli.Forwarder.Storage; +using SeqCli.Tests.Forwarder.Filesystem; +using Xunit; + +namespace SeqCli.Tests.Forwarder.Storage; + +public class BookmarkTests +{ + [Fact] + public void CreateSetGet() + { + var directory = new InMemoryStoreDirectory(); + + var bookmark = Bookmark.Open(directory); + + Assert.False(bookmark.TryGet(out var value)); + Assert.Null(value); + + Assert.True(bookmark.TrySet(new BookmarkValue(42, 1))); + Assert.True(bookmark.TryGet(out value)); + Assert.Equal(new BookmarkValue(42, 1), value.Value); + + Assert.True(bookmark.TrySet(new BookmarkValue(42, int.MaxValue))); + Assert.True(bookmark.TryGet(out value)); + Assert.Equal(new BookmarkValue(42, int.MaxValue), value.Value); + } + + [Fact] + public void OpenDeletesOldBookmarks() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create($"{1L:x16}.bookmark", new BookmarkValue(3, 3478).Encode()); + directory.Create($"{3L:x16}.bookmark", new BookmarkValue(42, 17).Encode()); + + Assert.Equal(2, directory.Files.Count); + + var bookmark = Bookmark.Open(directory); + + Assert.Equal($"{3L:x16}.bookmark", directory.Files.Single().Key); + + Assert.True(bookmark.TryGet(out var value)); + Assert.Equal(new BookmarkValue(42, 17), value); + } + + [Fact] + public void OpenDeletesCorruptedBookmarks() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create($"{1L:x16}.bookmark", new BookmarkValue(3, 3478).Encode()); + + // This bookmark is invalid + directory.Create($"{3L:x16}.bookmark", new byte[] { 1, 2, 3 }); + + var bookmark = Bookmark.Open(directory); + + Assert.Empty(directory.Files); + + Assert.True(bookmark.TrySet(new BookmarkValue(42, 17))); + + Assert.Equal($"{4L:x16}.bookmark", directory.Files.Single().Key); + } + + [Fact] + public void OpenDeletesMisnamedBookmarks() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create($"{1L:x16}.bookmark", new BookmarkValue(3, 3478).Encode()); + + // This bookmark is invalid + directory.Create($"ff{3L:x16}.bookmark", new BookmarkValue(42, 17).Encode()); + + var bookmark = Bookmark.Open(directory); + + Assert.Single(directory.Files); + + Assert.True(bookmark.TryGet(out var value)); + Assert.Equal(new BookmarkValue(3, 3478), value); + } +} diff --git a/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs b/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs new file mode 100644 index 00000000..a217615b --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs @@ -0,0 +1,283 @@ +using System.Linq; +using SeqCli.Forwarder.Storage; +using SeqCli.Tests.Forwarder.Filesystem; +using Xunit; + +namespace SeqCli.Tests.Forwarder.Storage; + +public class BufferTests +{ + [Fact] + public void OpenAppendRead() + { + var directory = new InMemoryStoreDirectory(); + + using var writer = BufferAppender.Open(directory); + var reader = BufferReader.Open(directory); + + Assert.Equal(0, directory.Files.Count); + + // Append a payload + Assert.True(writer.TryAppend("{\"id\":1}\n"u8.ToArray(), long.MaxValue)); + Assert.Equal(1, directory.Files.Count); + + // Read the payload + Assert.False(reader.TryFillBatch(10, out _)); + Assert.True(reader.TryFillBatch(10, out var batch)); + var batchBuffer = batch.Value; + Assert.Equal(new BufferReaderHead(1, 9), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":1}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + + // Advance the reader + reader.AdvanceTo(batchBuffer.ReaderHead); + Assert.False(reader.TryFillBatch(10, out batch)); + + // Append another payload + Assert.True(writer.TryAppend("{\"id\":2}\n"u8.ToArray(), long.MaxValue)); + Assert.Equal(1, directory.Files.Count); + + // Read the payload + Assert.True(reader.TryFillBatch(10, out batch)); + batchBuffer = batch.Value; + Assert.Equal(new BufferReaderHead(1, 18), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":2}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + + // Advance the reader + reader.AdvanceTo(batchBuffer.ReaderHead); + Assert.False(reader.TryFillBatch(10, out batch)); + } + + [Fact] + public void ReadWaitsUntilCompleteDataOnLastChunk() + { + var directory = new InMemoryStoreDirectory(); + + var chunk = directory.Create(new ChunkName(1).ToString(), "{\"id\":1"u8.ToArray()); + + var reader = BufferReader.Open(directory); + + Assert.False(reader.TryFillBatch(512, out _)); + + chunk.Append("}"u8.ToArray()); + + Assert.False(reader.TryFillBatch(512, out _)); + + chunk.Append("\n"u8.ToArray()); + + Assert.True(reader.TryFillBatch(512, out var batch)); + var batchBuffer = batch.Value; + + Assert.Equal(new BufferReaderHead(1, 9), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":1}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + } + + [Fact] + public void ReadDiscardsPreviouslyReadChunks() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create(new ChunkName(1).ToString(), "{\"id\":1}\n"u8.ToArray()); + directory.Create(new ChunkName(2).ToString(), "{\"id\":2}\n"u8.ToArray()); + + var reader = BufferReader.Open(directory); + + Assert.True(reader.TryFillBatch(512, out var batch)); + var batchBuffer = batch.Value; + Assert.Equal(new BufferReaderHead(2, 9), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":1}\n{\"id\":2}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + + Assert.Equal(2, directory.Files.Count); + + reader.AdvanceTo(batchBuffer.ReaderHead); + + Assert.Equal(1, directory.Files.Count); + + directory.Create(new ChunkName(1).ToString(), "{\"id\":1}\n"u8.ToArray()); + directory.Create(new ChunkName(3).ToString(), "{\"id\":3}\n"u8.ToArray()); + + Assert.Equal(3, directory.Files.Count); + + Assert.False(reader.TryFillBatch(512, out _)); + Assert.True(reader.TryFillBatch(512, out batch)); + batchBuffer = batch.Value; + Assert.Equal(new BufferReaderHead(3, 9), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":3}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + + reader.AdvanceTo(batchBuffer.ReaderHead); + + Assert.Equal(1, directory.Files.Count); + } + + [Fact] + public void ReadDiscardsOversizePayloads() + { + var directory = new InMemoryStoreDirectory(); + + using var writer = BufferAppender.Open(directory); + + Assert.True(writer.TryAppend("{\"id\":1}\n"u8.ToArray(), long.MaxValue)); + + var reader = BufferReader.Open(directory); + + Assert.False(reader.TryFillBatch(5, out _)); + Assert.False(reader.TryFillBatch(512, out _)); + } + + [Fact] + public void ReadDoesNotDiscardAcrossFiles() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create(new ChunkName(1).ToString(), "{\"id\":1}"u8.ToArray()); + directory.Create(new ChunkName(2).ToString(), "{\"id\":2}\n"u8.ToArray()); + + var reader = BufferReader.Open(directory); + + Assert.True(reader.TryFillBatch(512, out var batch)); + var batchBuffer = batch.Value; + + Assert.Equal(new BufferReaderHead(2, 9), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":2}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + } + + [Fact] + public void ReadStopsDiscardingOnExternalDelete() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create(new ChunkName(1).ToString(), "{\"id\":1}"u8.ToArray()); + + var reader = BufferReader.Open(directory); + + Assert.False(reader.TryFillBatch(5, out _)); + + // Deleting the file here will cause our discarding chunk to change + Assert.True(directory.TryDelete(new ChunkName(1).ToString())); + directory.Create(new ChunkName(2).ToString(), "{\"id\":2}\n"u8.ToArray()); + + Assert.False(reader.TryFillBatch(512, out _)); + Assert.True(reader.TryFillBatch(512, out var batch)); + var batchBuffer = batch.Value; + + Assert.Equal(new BufferReaderHead(2, 9), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":2}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + } + + [Fact] + public void ReadStopsDiscardingOnExternalCreate() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create(new ChunkName(1).ToString(), "{\"id\":1}"u8.ToArray()); + + var reader = BufferReader.Open(directory); + + Assert.False(reader.TryFillBatch(5, out _)); + + // Creating a new file here will cause a new one to be created + directory.Create(new ChunkName(2).ToString(), "{\"id\":2}\n"u8.ToArray()); + + Assert.False(reader.TryFillBatch(512, out _)); + Assert.True(reader.TryFillBatch(512, out var batch)); + var batchBuffer = batch.Value; + + Assert.Equal(new BufferReaderHead(2, 9), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":2}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + } + + [Fact] + public void AppendRolloverOnWrite() + { + var directory = new InMemoryStoreDirectory(); + + using var writer = BufferAppender.Open(directory); + var reader = BufferReader.Open(directory); + + Assert.Equal(0, directory.Files.Count); + + Assert.True(writer.TryAppend("{\"id\":1}\n"u8.ToArray(), 17)); + Assert.True(writer.TryAppend("{\"id\":2}\n"u8.ToArray(), 17)); + + Assert.Equal(1, directory.Files.Count); + + Assert.True(writer.TryAppend("{\"id\":3}\n"u8.ToArray(), 17)); + + Assert.Equal(2, directory.Files.Count); + + Assert.False(reader.TryFillBatch(512, out _)); + Assert.True(reader.TryFillBatch(512, out var batch)); + var batchBuffer = batch.Value; + + Assert.Equal(new BufferReaderHead(2, 9), batchBuffer.ReaderHead); + Assert.Equal("{\"id\":1}\n{\"id\":2}\n{\"id\":3}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + + reader.AdvanceTo(batchBuffer.ReaderHead); + + Assert.Equal(1, directory.Files.Count); + } + + [Fact] + public void ExistingFilesAreReadonly() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create(new ChunkName(0).ToString()); + + using var writer = BufferAppender.Open(directory); + var reader = BufferReader.Open(directory); + + Assert.Equal(1, directory.Files.Count); + + Assert.True(writer.TryAppend("{\"id\":1}\n"u8.ToArray(), long.MaxValue)); + + Assert.Equal(2, directory.Files.Count); + + Assert.False(reader.TryFillBatch(512, out _)); + Assert.True(reader.TryFillBatch(512, out var batch)); + var batchBuffer = batch.Value; + + Assert.Equal(new BufferReaderHead(1, 9), batchBuffer.ReaderHead); + } + + [Fact] + public void OpenReadAcrossChunks() + { + var directory = new InMemoryStoreDirectory(); + + directory.Create(new ChunkName(0).ToString(), "{\"id\":1}\n"u8.ToArray()); + directory.Create(new ChunkName(1).ToString(), "{\"id\":2}\n"u8.ToArray()); + directory.Create(new ChunkName(2).ToString(), "{\"id\":3}\n"u8.ToArray()); + + var reader = BufferReader.Open(directory); + + Assert.Equal(3, directory.Files.Count); + + Assert.True(reader.TryFillBatch(512, out var batch)); + var batchBuffer = batch.Value; + + Assert.Equal("{\"id\":1}\n{\"id\":2}\n{\"id\":3}\n"u8.ToArray(), batchBuffer.AsSpan().ToArray()); + + reader.AdvanceTo(batchBuffer.ReaderHead); + + Assert.Single(directory.Files); + } + + [Fact] + public void MaxChunksOnAppender() + { + var directory = new InMemoryStoreDirectory(); + + using var appender = BufferAppender.Open(directory); + + for (var i = 0; i < 10; i++) Assert.True(appender.TryAppend("{\"id\":1}\n"u8.ToArray(), 5, 3)); + + var files = directory.Files.Select(f => f.Key).ToList(); + files.Sort(); + + Assert.Equal([ + "0000000000000008.clef", + "0000000000000009.clef", + "000000000000000a.clef" + ], files); + } +} diff --git a/test/SeqCli.Tests/Forwarder/Storage/IdentifierTests.cs b/test/SeqCli.Tests/Forwarder/Storage/IdentifierTests.cs new file mode 100644 index 00000000..b5d1f752 --- /dev/null +++ b/test/SeqCli.Tests/Forwarder/Storage/IdentifierTests.cs @@ -0,0 +1,31 @@ +using SeqCli.Forwarder.Storage; +using Xunit; + +namespace SeqCli.Tests.Forwarder.Storage; + +public class IdentifierTests +{ + [Theory] + [InlineData("0000000000000000.clef", 0)] + [InlineData("0000000000000001.clef", 1)] + [InlineData("000000000000000a.clef", 10)] + [InlineData("ffffffffffffffff.clef", ulong.MaxValue)] + public void ParseValid(string name, ulong expected) + { + Assert.True(ChunkName.TryParse(name, out var actual)); + + Assert.Equal(expected, actual.Value.Id); + Assert.Equal(name, actual.Value.ToString()); + } + + [Theory] + [InlineData("0.clef")] + [InlineData("one.clef")] + [InlineData("00000000000.clef.value")] + [InlineData("0ffffffffffffffff.clef")] + [InlineData("0xffffffffffffff.clef")] + public void ParseInvalid(string name) + { + Assert.False(ChunkName.TryParse(name, out _)); + } +} diff --git a/test/SeqCli.Tests/SeqCli.Tests.csproj b/test/SeqCli.Tests/SeqCli.Tests.csproj index 9ebc2e63..9f75b8f4 100644 --- a/test/SeqCli.Tests/SeqCli.Tests.csproj +++ b/test/SeqCli.Tests/SeqCli.Tests.csproj @@ -15,6 +15,7 @@ + From aeb3b5f68bb4b4d054f6856edf80f380909a11ce Mon Sep 17 00:00:00 2001 From: KodrAus Date: Thu, 9 May 2024 09:13:48 +1000 Subject: [PATCH 3/7] fix up warnings in tests --- .../Forwarder/Filesystem/InMemoryStoreFile.cs | 2 ++ .../Forwarder/Storage/BufferTests.cs | 18 +++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs index 1a48b246..dd2b54e8 100644 --- a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs @@ -1,3 +1,5 @@ +#nullable enable + using System; using System.Diagnostics.CodeAnalysis; using SeqCli.Forwarder.Filesystem; diff --git a/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs b/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs index a217615b..86129275 100644 --- a/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs +++ b/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs @@ -15,11 +15,11 @@ public void OpenAppendRead() using var writer = BufferAppender.Open(directory); var reader = BufferReader.Open(directory); - Assert.Equal(0, directory.Files.Count); + Assert.Empty(directory.Files); // Append a payload Assert.True(writer.TryAppend("{\"id\":1}\n"u8.ToArray(), long.MaxValue)); - Assert.Equal(1, directory.Files.Count); + Assert.Single(directory.Files); // Read the payload Assert.False(reader.TryFillBatch(10, out _)); @@ -34,7 +34,7 @@ public void OpenAppendRead() // Append another payload Assert.True(writer.TryAppend("{\"id\":2}\n"u8.ToArray(), long.MaxValue)); - Assert.Equal(1, directory.Files.Count); + Assert.Single(directory.Files); // Read the payload Assert.True(reader.TryFillBatch(10, out batch)); @@ -90,7 +90,7 @@ public void ReadDiscardsPreviouslyReadChunks() reader.AdvanceTo(batchBuffer.ReaderHead); - Assert.Equal(1, directory.Files.Count); + Assert.Single(directory.Files); directory.Create(new ChunkName(1).ToString(), "{\"id\":1}\n"u8.ToArray()); directory.Create(new ChunkName(3).ToString(), "{\"id\":3}\n"u8.ToArray()); @@ -105,7 +105,7 @@ public void ReadDiscardsPreviouslyReadChunks() reader.AdvanceTo(batchBuffer.ReaderHead); - Assert.Equal(1, directory.Files.Count); + Assert.Single(directory.Files); } [Fact] @@ -193,12 +193,12 @@ public void AppendRolloverOnWrite() using var writer = BufferAppender.Open(directory); var reader = BufferReader.Open(directory); - Assert.Equal(0, directory.Files.Count); + Assert.Empty(directory.Files); Assert.True(writer.TryAppend("{\"id\":1}\n"u8.ToArray(), 17)); Assert.True(writer.TryAppend("{\"id\":2}\n"u8.ToArray(), 17)); - Assert.Equal(1, directory.Files.Count); + Assert.Single(directory.Files); Assert.True(writer.TryAppend("{\"id\":3}\n"u8.ToArray(), 17)); @@ -213,7 +213,7 @@ public void AppendRolloverOnWrite() reader.AdvanceTo(batchBuffer.ReaderHead); - Assert.Equal(1, directory.Files.Count); + Assert.Single(directory.Files); } [Fact] @@ -226,7 +226,7 @@ public void ExistingFilesAreReadonly() using var writer = BufferAppender.Open(directory); var reader = BufferReader.Open(directory); - Assert.Equal(1, directory.Files.Count); + Assert.Single(directory.Files); Assert.True(writer.TryAppend("{\"id\":1}\n"u8.ToArray(), long.MaxValue)); From 6ce99a46d35ac02fb846d58ba6954b10728b6bf0 Mon Sep 17 00:00:00 2001 From: KodrAus Date: Thu, 9 May 2024 09:29:35 +1000 Subject: [PATCH 4/7] remove extra csproj item --- test/SeqCli.Tests/SeqCli.Tests.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/test/SeqCli.Tests/SeqCli.Tests.csproj b/test/SeqCli.Tests/SeqCli.Tests.csproj index e46c0ee0..1bf867fa 100644 --- a/test/SeqCli.Tests/SeqCli.Tests.csproj +++ b/test/SeqCli.Tests/SeqCli.Tests.csproj @@ -15,7 +15,6 @@ - From cc8f67a942791645cb81001302e82418a317a201 Mon Sep 17 00:00:00 2001 From: KodrAus Date: Thu, 9 May 2024 10:19:49 +1000 Subject: [PATCH 5/7] fix up windows build --- Build.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Build.ps1 b/Build.ps1 index 29f5ea0f..efd19536 100644 --- a/Build.ps1 +++ b/Build.ps1 @@ -32,7 +32,7 @@ function Create-ArtifactDir function Publish-Archives($version) { - $rids = $([xml](Get-Content .\src\SeqCli\SeqCli.csproj)).Project.PropertyGroup.RuntimeIdentifiers.Split(';') + $rids = $([xml](Get-Content .\src\SeqCli\SeqCli.csproj)).Project.PropertyGroup.RuntimeIdentifiers[0].Split(';') foreach ($rid in $rids) { $tfm = $framework if ($rid -eq "win-x64") { From a8ba8430a88cb0919ecc4bffc3a942729116d7e0 Mon Sep 17 00:00:00 2001 From: KodrAus Date: Thu, 9 May 2024 15:29:33 +1000 Subject: [PATCH 6/7] fix up tfms --- src/SeqCli/SeqCli.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index 61f7f087..c15f93fb 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -1,7 +1,7 @@  Exe - net8.0 + net8.0;net8.0-windows seqcli ..\..\asset\SeqCli.ico win-x64;linux-x64;linux-musl-x64;osx-x64;linux-arm64;linux-musl-arm64;osx-arm64 From b8b6171d68b60368b7082e00eda9f5d606901bfc Mon Sep 17 00:00:00 2001 From: Ashley Mannix Date: Fri, 10 May 2024 14:27:21 +1000 Subject: [PATCH 7/7] updates based on PR review --- src/SeqCli/Apps/Hosting/AppContainer.cs | 2 +- src/SeqCli/Cli/Commands/App/InstallCommand.cs | 2 +- src/SeqCli/Cli/Commands/App/UpdateCommand.cs | 2 +- .../Commands/Bench/BenchCasesCollection.cs | 2 +- src/SeqCli/Cli/Commands/Bench/BenchCommand.cs | 2 +- .../Cli/Commands/Bench/QueryBenchCase.cs | 2 +- .../Commands/Bench/QueryBenchCaseTimings.cs | 2 +- .../Cli/Commands/Forwarder/StartCommand.cs | 2 +- .../Cli/Commands/Forwarder/StopCommand.cs | 2 +- .../Commands/Forwarder/UninstallCommand.cs | 2 +- src/SeqCli/Cli/Commands/Node/DemoteCommand.cs | 2 +- src/SeqCli/Cli/Commands/Node/HealthCommand.cs | 2 +- src/SeqCli/Cli/Commands/Node/ListCommand.cs | 2 +- .../Cli/Commands/Sample/IngestCommand.cs | 2 +- .../Cli/Commands/Sample/SetupCommand.cs | 2 +- .../Cli/Commands/Settings/ClearCommand.cs | 2 +- .../Cli/Commands/Settings/NamesCommand.cs | 2 +- .../Cli/Commands/Settings/SetCommand.cs | 2 +- .../Cli/Commands/Settings/ShowCommand.cs | 2 +- src/SeqCli/Cli/Features/TimeoutFeature.cs | 2 +- .../Filesystem/EmptyStoreFileReader.cs | 4 +-- .../Forwarder/Filesystem/StoreDirectory.cs | 4 +-- src/SeqCli/Forwarder/Filesystem/StoreFile.cs | 4 +-- .../Forwarder/Filesystem/StoreFileAppender.cs | 4 +-- .../Forwarder/Filesystem/StoreFileReader.cs | 4 +-- .../Filesystem/System/SystemStoreDirectory.cs | 11 +++++--- .../Filesystem/System/SystemStoreFile.cs | 4 +-- .../System/SystemStoreFileAppender.cs | 4 +-- .../System/SystemStoreFileReader.cs | 4 +-- .../Forwarder/Filesystem/System/Unix/Libc.cs | 6 +++-- src/SeqCli/Forwarder/ForwarderModule.cs | 2 +- src/SeqCli/Forwarder/Storage/Bookmark.cs | 4 +-- src/SeqCli/Forwarder/Storage/BookmarkName.cs | 4 +-- src/SeqCli/Forwarder/Storage/BookmarkValue.cs | 4 +-- .../Forwarder/Storage/BufferAppender.cs | 25 +++++++++++++------ .../Forwarder/Storage/BufferAppenderChunk.cs | 10 ++++++-- src/SeqCli/Forwarder/Storage/BufferReader.cs | 4 +-- .../Forwarder/Storage/BufferReaderBatch.cs | 4 +-- .../Forwarder/Storage/BufferReaderChunk.cs | 13 ++++++++-- .../Storage/BufferReaderChunkHead.cs | 4 +-- .../Forwarder/Storage/BufferReaderHead.cs | 4 +-- src/SeqCli/Forwarder/Storage/ChunkName.cs | 4 +-- src/SeqCli/Forwarder/Storage/Identifier.cs | 4 +-- .../Forwarder/Web/Api/ApiRootEndpoints.cs | 2 +- .../Forwarder/Web/Api/IngestionEndpoints.cs | 2 +- .../Web/Api/IngestionLogEndpoints.cs | 2 +- .../Forwarder/Web/Host/ServerService.cs | 2 +- src/SeqCli/Sample/Loader/Simulation.cs | 2 +- src/SeqCli/SeqCli.csproj | 3 +++ src/SeqCli/Templates/Ast/JsonTemplate.cs | 2 +- src/SeqCli/Templates/Ast/JsonTemplateArray.cs | 2 +- .../Templates/Ast/JsonTemplateBoolean.cs | 2 +- src/SeqCli/Templates/Ast/JsonTemplateCall.cs | 2 +- src/SeqCli/Templates/Ast/JsonTemplateNull.cs | 2 +- .../Templates/Ast/JsonTemplateNumber.cs | 2 +- .../Templates/Ast/JsonTemplateObject.cs | 2 +- .../Templates/Ast/JsonTemplateString.cs | 2 +- .../Evaluator/JsonTemplateEvaluator.cs | 2 +- .../Evaluator/JsonTemplateFunction.cs | 2 +- src/SeqCli/Templates/Import/EntityTemplate.cs | 2 +- .../Templates/Import/EntityTemplateLoader.cs | 2 +- src/SeqCli/Templates/Import/GenericEntity.cs | 2 +- .../Templates/Import/TemplateSetImporter.cs | 2 +- .../JsonTemplateObjectGraphConverter.cs | 2 +- .../Templates/Parser/JsonTemplateParser.cs | 2 +- .../Parser/JsonTemplateTextParsers.cs | 2 +- .../Templates/Parser/JsonTemplateToken.cs | 2 +- .../Templates/Parser/JsonTemplateTokenizer.cs | 2 +- src/SeqCli/Util/LogEventPropertyFactory.cs | 2 +- .../Filesystem/InMemoryStoreDirectory.cs | 4 +-- .../Forwarder/Filesystem/InMemoryStoreFile.cs | 2 +- .../Filesystem/InMemoryStoreFileAppender.cs | 2 +- .../Filesystem/InMemoryStoreFileReader.cs | 2 +- 73 files changed, 136 insertions(+), 102 deletions(-) diff --git a/src/SeqCli/Apps/Hosting/AppContainer.cs b/src/SeqCli/Apps/Hosting/AppContainer.cs index fe23a843..72184492 100644 --- a/src/SeqCli/Apps/Hosting/AppContainer.cs +++ b/src/SeqCli/Apps/Hosting/AppContainer.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/App/InstallCommand.cs b/src/SeqCli/Cli/Commands/App/InstallCommand.cs index 94d98482..7f9974d8 100644 --- a/src/SeqCli/Cli/Commands/App/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/App/InstallCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/App/UpdateCommand.cs b/src/SeqCli/Cli/Commands/App/UpdateCommand.cs index bf7fb40f..e0051a9c 100644 --- a/src/SeqCli/Cli/Commands/App/UpdateCommand.cs +++ b/src/SeqCli/Cli/Commands/App/UpdateCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Bench/BenchCasesCollection.cs b/src/SeqCli/Cli/Commands/Bench/BenchCasesCollection.cs index 441b6ae0..a47c8206 100644 --- a/src/SeqCli/Cli/Commands/Bench/BenchCasesCollection.cs +++ b/src/SeqCli/Cli/Commands/Bench/BenchCasesCollection.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Bench/BenchCommand.cs b/src/SeqCli/Cli/Commands/Bench/BenchCommand.cs index 8742d77b..0b538815 100644 --- a/src/SeqCli/Cli/Commands/Bench/BenchCommand.cs +++ b/src/SeqCli/Cli/Commands/Bench/BenchCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Bench/QueryBenchCase.cs b/src/SeqCli/Cli/Commands/Bench/QueryBenchCase.cs index ce31c26e..55bee6d2 100644 --- a/src/SeqCli/Cli/Commands/Bench/QueryBenchCase.cs +++ b/src/SeqCli/Cli/Commands/Bench/QueryBenchCase.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Bench/QueryBenchCaseTimings.cs b/src/SeqCli/Cli/Commands/Bench/QueryBenchCaseTimings.cs index a3bc8834..be8cd5ec 100644 --- a/src/SeqCli/Cli/Commands/Bench/QueryBenchCaseTimings.cs +++ b/src/SeqCli/Cli/Commands/Bench/QueryBenchCaseTimings.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs index 66f859fc..0f0e97d6 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs index 955c550c..14c9c9e7 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs index 50845d26..6805843e 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Node/DemoteCommand.cs b/src/SeqCli/Cli/Commands/Node/DemoteCommand.cs index 382d06a5..68358710 100644 --- a/src/SeqCli/Cli/Commands/Node/DemoteCommand.cs +++ b/src/SeqCli/Cli/Commands/Node/DemoteCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Node/HealthCommand.cs b/src/SeqCli/Cli/Commands/Node/HealthCommand.cs index 24dad8f1..0b82b63f 100644 --- a/src/SeqCli/Cli/Commands/Node/HealthCommand.cs +++ b/src/SeqCli/Cli/Commands/Node/HealthCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Node/ListCommand.cs b/src/SeqCli/Cli/Commands/Node/ListCommand.cs index 9b24b03c..1a69ee1a 100644 --- a/src/SeqCli/Cli/Commands/Node/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/Node/ListCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Sample/IngestCommand.cs b/src/SeqCli/Cli/Commands/Sample/IngestCommand.cs index ff94f887..95644de5 100644 --- a/src/SeqCli/Cli/Commands/Sample/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/Sample/IngestCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Sample/SetupCommand.cs b/src/SeqCli/Cli/Commands/Sample/SetupCommand.cs index 8905c97c..71f5c952 100644 --- a/src/SeqCli/Cli/Commands/Sample/SetupCommand.cs +++ b/src/SeqCli/Cli/Commands/Sample/SetupCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Settings/ClearCommand.cs b/src/SeqCli/Cli/Commands/Settings/ClearCommand.cs index e4025de8..2a3ac781 100644 --- a/src/SeqCli/Cli/Commands/Settings/ClearCommand.cs +++ b/src/SeqCli/Cli/Commands/Settings/ClearCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Settings/NamesCommand.cs b/src/SeqCli/Cli/Commands/Settings/NamesCommand.cs index 40f060d5..d50345ab 100644 --- a/src/SeqCli/Cli/Commands/Settings/NamesCommand.cs +++ b/src/SeqCli/Cli/Commands/Settings/NamesCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Settings/SetCommand.cs b/src/SeqCli/Cli/Commands/Settings/SetCommand.cs index 67ad8844..8270f87e 100644 --- a/src/SeqCli/Cli/Commands/Settings/SetCommand.cs +++ b/src/SeqCli/Cli/Commands/Settings/SetCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Commands/Settings/ShowCommand.cs b/src/SeqCli/Cli/Commands/Settings/ShowCommand.cs index e351b41d..38582e80 100644 --- a/src/SeqCli/Cli/Commands/Settings/ShowCommand.cs +++ b/src/SeqCli/Cli/Commands/Settings/ShowCommand.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Cli/Features/TimeoutFeature.cs b/src/SeqCli/Cli/Features/TimeoutFeature.cs index d5fb066b..3530812e 100644 --- a/src/SeqCli/Cli/Features/TimeoutFeature.cs +++ b/src/SeqCli/Cli/Features/TimeoutFeature.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Forwarder/Filesystem/EmptyStoreFileReader.cs b/src/SeqCli/Forwarder/Filesystem/EmptyStoreFileReader.cs index 0410e23a..bc8ac092 100644 --- a/src/SeqCli/Forwarder/Filesystem/EmptyStoreFileReader.cs +++ b/src/SeqCli/Forwarder/Filesystem/EmptyStoreFileReader.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ namespace SeqCli.Forwarder.Filesystem; -public sealed class EmptyStoreFileReader : StoreFileReader +sealed class EmptyStoreFileReader : StoreFileReader { public override void Dispose() { diff --git a/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs b/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs index e9503162..39008865 100644 --- a/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs +++ b/src/SeqCli/Forwarder/Filesystem/StoreDirectory.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ namespace SeqCli.Forwarder.Filesystem; /// /// A container of s and their names. /// -public abstract class StoreDirectory +abstract class StoreDirectory { /// /// Create a new file with the given name, linking it into the filesystem. diff --git a/src/SeqCli/Forwarder/Filesystem/StoreFile.cs b/src/SeqCli/Forwarder/Filesystem/StoreFile.cs index 9e998523..1beed35f 100644 --- a/src/SeqCli/Forwarder/Filesystem/StoreFile.cs +++ b/src/SeqCli/Forwarder/Filesystem/StoreFile.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ namespace SeqCli.Forwarder.Filesystem; -public abstract class StoreFile +abstract class StoreFile { /// /// Get the length of this file. diff --git a/src/SeqCli/Forwarder/Filesystem/StoreFileAppender.cs b/src/SeqCli/Forwarder/Filesystem/StoreFileAppender.cs index 8e04dd91..31f7af4d 100644 --- a/src/SeqCli/Forwarder/Filesystem/StoreFileAppender.cs +++ b/src/SeqCli/Forwarder/Filesystem/StoreFileAppender.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ namespace SeqCli.Forwarder.Filesystem; -public abstract class StoreFileAppender : IDisposable +abstract class StoreFileAppender : IDisposable { public abstract void Dispose(); diff --git a/src/SeqCli/Forwarder/Filesystem/StoreFileReader.cs b/src/SeqCli/Forwarder/Filesystem/StoreFileReader.cs index d98feaad..5679c914 100644 --- a/src/SeqCli/Forwarder/Filesystem/StoreFileReader.cs +++ b/src/SeqCli/Forwarder/Filesystem/StoreFileReader.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ namespace SeqCli.Forwarder.Filesystem; -public abstract class StoreFileReader : IDisposable +abstract class StoreFileReader : IDisposable { public abstract void Dispose(); diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs index be27c3d4..e436185f 100644 --- a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,11 +16,14 @@ using System.Collections.Generic; using System.IO; using System.Runtime.InteropServices; + +#if UNIX using SeqCli.Forwarder.Filesystem.System.Unix; +#endif namespace SeqCli.Forwarder.Filesystem.System; -public sealed class SystemStoreDirectory : StoreDirectory +sealed class SystemStoreDirectory : StoreDirectory { readonly string _directoryPath; @@ -110,8 +113,7 @@ public override StoreFile ReplaceContents(string name, Span contents, bool static void Dirsync(string directoryPath) { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return; - +#if UNIX var dir = Libc.open(directoryPath, 0); if (dir == -1) return; @@ -121,5 +123,6 @@ static void Dirsync(string directoryPath) Libc.fsync(dir); Libc.close(dir); #pragma warning restore CA1806 +#endif } } diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFile.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFile.cs index 361b6741..55b37375 100644 --- a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFile.cs +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFile.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ namespace SeqCli.Forwarder.Filesystem.System; -public sealed class SystemStoreFile : StoreFile +sealed class SystemStoreFile : StoreFile { static readonly FileStreamOptions AppendOptions = new() { diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileAppender.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileAppender.cs index b64b9d26..298ea8d2 100644 --- a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileAppender.cs +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileAppender.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ namespace SeqCli.Forwarder.Filesystem.System; -public sealed class SystemStoreFileAppender : StoreFileAppender +sealed class SystemStoreFileAppender : StoreFileAppender { readonly FileStream _file; long _initialLength; diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileReader.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileReader.cs index 23e95101..22108eb5 100644 --- a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileReader.cs +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreFileReader.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ namespace SeqCli.Forwarder.Filesystem.System; -public sealed class SystemStoreFileReader : StoreFileReader +sealed class SystemStoreFileReader : StoreFileReader { readonly MemoryMappedViewAccessor _accessor; readonly MemoryMappedFile _file; diff --git a/src/SeqCli/Forwarder/Filesystem/System/Unix/Libc.cs b/src/SeqCli/Forwarder/Filesystem/System/Unix/Libc.cs index c9edc557..4561a20a 100644 --- a/src/SeqCli/Forwarder/Filesystem/System/Unix/Libc.cs +++ b/src/SeqCli/Forwarder/Filesystem/System/Unix/Libc.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#if UNIX using System.Runtime.InteropServices; namespace SeqCli.Forwarder.Filesystem.System.Unix; @@ -26,4 +27,5 @@ static class Libc [DllImport("libc")] public static extern int fsync(int fd); -} \ No newline at end of file +} +#endif diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 03d24974..0cc9c954 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Forwarder/Storage/Bookmark.cs b/src/SeqCli/Forwarder/Storage/Bookmark.cs index f3ec3ea4..3f75fb8b 100644 --- a/src/SeqCli/Forwarder/Storage/Bookmark.cs +++ b/src/SeqCli/Forwarder/Storage/Bookmark.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ namespace SeqCli.Forwarder.Storage; /// /// A durable bookmark of progress processing buffers. /// -public sealed class Bookmark +sealed class Bookmark { readonly StoreDirectory _storeDirectory; diff --git a/src/SeqCli/Forwarder/Storage/BookmarkName.cs b/src/SeqCli/Forwarder/Storage/BookmarkName.cs index 76206472..e40a8d8f 100644 --- a/src/SeqCli/Forwarder/Storage/BookmarkName.cs +++ b/src/SeqCli/Forwarder/Storage/BookmarkName.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ namespace SeqCli.Forwarder.Storage; /// /// A bookmark file name with its incrementing identifier. /// -public readonly record struct BookmarkName +readonly record struct BookmarkName { readonly string _name; diff --git a/src/SeqCli/Forwarder/Storage/BookmarkValue.cs b/src/SeqCli/Forwarder/Storage/BookmarkValue.cs index 72716391..7d86f90b 100644 --- a/src/SeqCli/Forwarder/Storage/BookmarkValue.cs +++ b/src/SeqCli/Forwarder/Storage/BookmarkValue.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ namespace SeqCli.Forwarder.Storage; /// /// The in-memory value of a bookmark. /// -public readonly record struct BookmarkValue(ulong Id, long CommitHead) +readonly record struct BookmarkValue(ulong Id, long CommitHead) { public void EncodeTo(Span bookmark) { diff --git a/src/SeqCli/Forwarder/Storage/BufferAppender.cs b/src/SeqCli/Forwarder/Storage/BufferAppender.cs index b88576fd..bc2ca6a6 100644 --- a/src/SeqCli/Forwarder/Storage/BufferAppender.cs +++ b/src/SeqCli/Forwarder/Storage/BufferAppender.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ namespace SeqCli.Forwarder.Storage; /// /// The write-side of a buffer. /// -public sealed class BufferAppender : IDisposable +sealed class BufferAppender : IDisposable { readonly StoreDirectory _storeDirectory; BufferAppenderChunk? _currentChunk; @@ -69,19 +69,30 @@ public bool TryAppend(Span batch, long targetChunkLength, int? maxChunks = { if (batch.Length == 0) return true; - if (batch[^1] != (byte)'\n') throw new Exception("Batches must end with a newline character (\\n)"); + if (batch[^1] != (byte)'\n') throw new Exception("Batches must end with a newline character (\\n)."); if (_currentChunk != null) + { // Only use the existing chunk if it's writable and shouldn't be rolled over if (_currentChunk.WriteHead > targetChunkLength) { // Run a sync before moving to a new file, just to make sure any // buffered data makes its way to disk - _currentChunk.Appender.Sync(); - - _currentChunk.Dispose(); - _currentChunk = null; + try + { + _currentChunk.Appender.Sync(); + } + catch (IOException) + { + // Ignored + } + finally + { + _currentChunk.Dispose(); + _currentChunk = null; + } } + } // If there's no suitable candidate chunk then create a new one if (_currentChunk == null) diff --git a/src/SeqCli/Forwarder/Storage/BufferAppenderChunk.cs b/src/SeqCli/Forwarder/Storage/BufferAppenderChunk.cs index f68acd67..69156679 100644 --- a/src/SeqCli/Forwarder/Storage/BufferAppenderChunk.cs +++ b/src/SeqCli/Forwarder/Storage/BufferAppenderChunk.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,8 +17,14 @@ namespace SeqCli.Forwarder.Storage; -sealed record BufferAppenderChunk(StoreFileAppender Appender) : IDisposable +class BufferAppenderChunk : IDisposable { + public BufferAppenderChunk(StoreFileAppender appender) + { + Appender = appender; + } + + public StoreFileAppender Appender { get; } public long WriteHead { get; set; } public void Dispose() diff --git a/src/SeqCli/Forwarder/Storage/BufferReader.cs b/src/SeqCli/Forwarder/Storage/BufferReader.cs index f2746b8c..6fba78e9 100644 --- a/src/SeqCli/Forwarder/Storage/BufferReader.cs +++ b/src/SeqCli/Forwarder/Storage/BufferReader.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace SeqCli.Forwarder.Storage; /// /// The read-side of a buffer. /// -public sealed class BufferReader +sealed class BufferReader { readonly StoreDirectory _storeDirectory; BufferReaderHead? _discardingHead; diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs b/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs index 069c7535..697b98e4 100644 --- a/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs +++ b/src/SeqCli/Forwarder/Storage/BufferReaderBatch.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ namespace SeqCli.Forwarder.Storage; /// /// A contiguous batch of records pulled from a reader. /// -public readonly record struct BufferReaderBatch +readonly record struct BufferReaderBatch { readonly int _length; diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderChunk.cs b/src/SeqCli/Forwarder/Storage/BufferReaderChunk.cs index cf7f9b0c..e957f3fc 100644 --- a/src/SeqCli/Forwarder/Storage/BufferReaderChunk.cs +++ b/src/SeqCli/Forwarder/Storage/BufferReaderChunk.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,8 +20,17 @@ namespace SeqCli.Forwarder.Storage; /// /// An active chunk in a . /// -record BufferReaderChunk(ChunkName Name, StoreFile Chunk) : IDisposable +class BufferReaderChunk : IDisposable { + public BufferReaderChunk(ChunkName name, StoreFile chunk) + { + Name = name; + Chunk = chunk; + } + + public ChunkName Name { get; } + public StoreFile Chunk { get; } + (long, StoreFileReader)? _reader; public void Dispose() diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderChunkHead.cs b/src/SeqCli/Forwarder/Storage/BufferReaderChunkHead.cs index a1778342..7969d254 100644 --- a/src/SeqCli/Forwarder/Storage/BufferReaderChunkHead.cs +++ b/src/SeqCli/Forwarder/Storage/BufferReaderChunkHead.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ namespace SeqCli.Forwarder.Storage; /// /// The current position in a . /// -public readonly record struct BufferReaderChunkHead(long CommitHead, long WriteHead) +readonly record struct BufferReaderChunkHead(long CommitHead, long WriteHead) { public long Unadvanced => WriteHead - CommitHead; } diff --git a/src/SeqCli/Forwarder/Storage/BufferReaderHead.cs b/src/SeqCli/Forwarder/Storage/BufferReaderHead.cs index 0240fa51..f1f34217 100644 --- a/src/SeqCli/Forwarder/Storage/BufferReaderHead.cs +++ b/src/SeqCli/Forwarder/Storage/BufferReaderHead.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,4 +17,4 @@ namespace SeqCli.Forwarder.Storage; /// /// A position in a . /// -public readonly record struct BufferReaderHead(ulong Chunk, long CommitHead); +readonly record struct BufferReaderHead(ulong Chunk, long CommitHead); diff --git a/src/SeqCli/Forwarder/Storage/ChunkName.cs b/src/SeqCli/Forwarder/Storage/ChunkName.cs index fc301cf4..dcc85a78 100644 --- a/src/SeqCli/Forwarder/Storage/ChunkName.cs +++ b/src/SeqCli/Forwarder/Storage/ChunkName.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ namespace SeqCli.Forwarder.Storage; /// /// A chunk file name with its incrementing identifier. /// -public readonly record struct ChunkName +readonly record struct ChunkName { readonly string _name; diff --git a/src/SeqCli/Forwarder/Storage/Identifier.cs b/src/SeqCli/Forwarder/Storage/Identifier.cs index 000e0f07..0f0ab980 100644 --- a/src/SeqCli/Forwarder/Storage/Identifier.cs +++ b/src/SeqCli/Forwarder/Storage/Identifier.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ namespace SeqCli.Forwarder.Storage; /// /// Utilities for parsing and formatting file names with sortable identifiers. /// -public static class Identifier +static class Identifier { /// /// Try parse the identifier from the given name with the given extension. diff --git a/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs index 822ecb23..60b418f6 100644 --- a/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index e98c3828..78194d2e 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs index 2cbb3f8f..b9fe5686 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Forwarder/Web/Host/ServerService.cs b/src/SeqCli/Forwarder/Web/Host/ServerService.cs index 6832e953..a8fb7e75 100644 --- a/src/SeqCli/Forwarder/Web/Host/ServerService.cs +++ b/src/SeqCli/Forwarder/Web/Host/ServerService.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd +// Copyright © Datalust Pty Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Sample/Loader/Simulation.cs b/src/SeqCli/Sample/Loader/Simulation.cs index fba0a939..196ea265 100644 --- a/src/SeqCli/Sample/Loader/Simulation.cs +++ b/src/SeqCli/Sample/Loader/Simulation.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index c15f93fb..97479768 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -30,6 +30,9 @@ LINUX + + UNIX + diff --git a/src/SeqCli/Templates/Ast/JsonTemplate.cs b/src/SeqCli/Templates/Ast/JsonTemplate.cs index 4105f2de..06d74822 100644 --- a/src/SeqCli/Templates/Ast/JsonTemplate.cs +++ b/src/SeqCli/Templates/Ast/JsonTemplate.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Ast/JsonTemplateArray.cs b/src/SeqCli/Templates/Ast/JsonTemplateArray.cs index d7224419..cfd63080 100644 --- a/src/SeqCli/Templates/Ast/JsonTemplateArray.cs +++ b/src/SeqCli/Templates/Ast/JsonTemplateArray.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Ast/JsonTemplateBoolean.cs b/src/SeqCli/Templates/Ast/JsonTemplateBoolean.cs index a6599a64..1fc1bf2e 100644 --- a/src/SeqCli/Templates/Ast/JsonTemplateBoolean.cs +++ b/src/SeqCli/Templates/Ast/JsonTemplateBoolean.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Ast/JsonTemplateCall.cs b/src/SeqCli/Templates/Ast/JsonTemplateCall.cs index ddcb93bc..1f1cd516 100644 --- a/src/SeqCli/Templates/Ast/JsonTemplateCall.cs +++ b/src/SeqCli/Templates/Ast/JsonTemplateCall.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Ast/JsonTemplateNull.cs b/src/SeqCli/Templates/Ast/JsonTemplateNull.cs index dd9cda6c..01a44a6e 100644 --- a/src/SeqCli/Templates/Ast/JsonTemplateNull.cs +++ b/src/SeqCli/Templates/Ast/JsonTemplateNull.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Ast/JsonTemplateNumber.cs b/src/SeqCli/Templates/Ast/JsonTemplateNumber.cs index bb149866..32887fc7 100644 --- a/src/SeqCli/Templates/Ast/JsonTemplateNumber.cs +++ b/src/SeqCli/Templates/Ast/JsonTemplateNumber.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Ast/JsonTemplateObject.cs b/src/SeqCli/Templates/Ast/JsonTemplateObject.cs index 0c5be41b..fe11aac5 100644 --- a/src/SeqCli/Templates/Ast/JsonTemplateObject.cs +++ b/src/SeqCli/Templates/Ast/JsonTemplateObject.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Ast/JsonTemplateString.cs b/src/SeqCli/Templates/Ast/JsonTemplateString.cs index 96d2e8d6..d7896871 100644 --- a/src/SeqCli/Templates/Ast/JsonTemplateString.cs +++ b/src/SeqCli/Templates/Ast/JsonTemplateString.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Evaluator/JsonTemplateEvaluator.cs b/src/SeqCli/Templates/Evaluator/JsonTemplateEvaluator.cs index 4bb07eba..977b476e 100644 --- a/src/SeqCli/Templates/Evaluator/JsonTemplateEvaluator.cs +++ b/src/SeqCli/Templates/Evaluator/JsonTemplateEvaluator.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Evaluator/JsonTemplateFunction.cs b/src/SeqCli/Templates/Evaluator/JsonTemplateFunction.cs index 173bacd6..9d06abb9 100644 --- a/src/SeqCli/Templates/Evaluator/JsonTemplateFunction.cs +++ b/src/SeqCli/Templates/Evaluator/JsonTemplateFunction.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Import/EntityTemplate.cs b/src/SeqCli/Templates/Import/EntityTemplate.cs index eb99d3e9..fa08bde5 100644 --- a/src/SeqCli/Templates/Import/EntityTemplate.cs +++ b/src/SeqCli/Templates/Import/EntityTemplate.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Import/EntityTemplateLoader.cs b/src/SeqCli/Templates/Import/EntityTemplateLoader.cs index 39a0f857..92975cdf 100644 --- a/src/SeqCli/Templates/Import/EntityTemplateLoader.cs +++ b/src/SeqCli/Templates/Import/EntityTemplateLoader.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Import/GenericEntity.cs b/src/SeqCli/Templates/Import/GenericEntity.cs index 48838362..b9f93650 100644 --- a/src/SeqCli/Templates/Import/GenericEntity.cs +++ b/src/SeqCli/Templates/Import/GenericEntity.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Import/TemplateSetImporter.cs b/src/SeqCli/Templates/Import/TemplateSetImporter.cs index 399e4bcf..9f14ec5c 100644 --- a/src/SeqCli/Templates/Import/TemplateSetImporter.cs +++ b/src/SeqCli/Templates/Import/TemplateSetImporter.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/ObjectGraphs/JsonTemplateObjectGraphConverter.cs b/src/SeqCli/Templates/ObjectGraphs/JsonTemplateObjectGraphConverter.cs index 9236e260..c23c8d2c 100644 --- a/src/SeqCli/Templates/ObjectGraphs/JsonTemplateObjectGraphConverter.cs +++ b/src/SeqCli/Templates/ObjectGraphs/JsonTemplateObjectGraphConverter.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Parser/JsonTemplateParser.cs b/src/SeqCli/Templates/Parser/JsonTemplateParser.cs index 5a24778f..a0cdd15b 100644 --- a/src/SeqCli/Templates/Parser/JsonTemplateParser.cs +++ b/src/SeqCli/Templates/Parser/JsonTemplateParser.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Parser/JsonTemplateTextParsers.cs b/src/SeqCli/Templates/Parser/JsonTemplateTextParsers.cs index cf28a346..41746e39 100644 --- a/src/SeqCli/Templates/Parser/JsonTemplateTextParsers.cs +++ b/src/SeqCli/Templates/Parser/JsonTemplateTextParsers.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Parser/JsonTemplateToken.cs b/src/SeqCli/Templates/Parser/JsonTemplateToken.cs index 289b48b1..fb375455 100644 --- a/src/SeqCli/Templates/Parser/JsonTemplateToken.cs +++ b/src/SeqCli/Templates/Parser/JsonTemplateToken.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Templates/Parser/JsonTemplateTokenizer.cs b/src/SeqCli/Templates/Parser/JsonTemplateTokenizer.cs index ed3c01f1..b2da6ca4 100644 --- a/src/SeqCli/Templates/Parser/JsonTemplateTokenizer.cs +++ b/src/SeqCli/Templates/Parser/JsonTemplateTokenizer.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/SeqCli/Util/LogEventPropertyFactory.cs b/src/SeqCli/Util/LogEventPropertyFactory.cs index 6b2a004d..89c23987 100644 --- a/src/SeqCli/Util/LogEventPropertyFactory.cs +++ b/src/SeqCli/Util/LogEventPropertyFactory.cs @@ -1,4 +1,4 @@ -// Copyright Datalust Pty Ltd and Contributors +// Copyright © Datalust Pty Ltd and Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreDirectory.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreDirectory.cs index 7382bc1d..00b1762d 100644 --- a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreDirectory.cs +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreDirectory.cs @@ -5,7 +5,7 @@ namespace SeqCli.Tests.Forwarder.Filesystem; -public class InMemoryStoreDirectory : StoreDirectory +class InMemoryStoreDirectory : StoreDirectory { readonly Dictionary _files = new(); @@ -13,7 +13,7 @@ public class InMemoryStoreDirectory : StoreDirectory public override InMemoryStoreFile Create(string name) { - if (_files.ContainsKey(name)) throw new Exception($"The file {name} already exists"); + if (_files.ContainsKey(name)) throw new Exception($"The file {name} already exists."); _files.Add(name, new InMemoryStoreFile()); diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs index dd2b54e8..26be3beb 100644 --- a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFile.cs @@ -6,7 +6,7 @@ namespace SeqCli.Tests.Forwarder.Filesystem; -public class InMemoryStoreFile : StoreFile +class InMemoryStoreFile : StoreFile { public byte[] Contents { get; private set; } = Array.Empty(); diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileAppender.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileAppender.cs index aeaf66e1..9b078eec 100644 --- a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileAppender.cs +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileAppender.cs @@ -4,7 +4,7 @@ namespace SeqCli.Tests.Forwarder.Filesystem; -public class InMemoryStoreFileAppender : StoreFileAppender +class InMemoryStoreFileAppender : StoreFileAppender { readonly List _incoming; diff --git a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileReader.cs b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileReader.cs index 56bd0838..d8507e6c 100644 --- a/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileReader.cs +++ b/test/SeqCli.Tests/Forwarder/Filesystem/InMemoryStoreFileReader.cs @@ -3,7 +3,7 @@ namespace SeqCli.Tests.Forwarder.Filesystem; -public class InMemoryStoreFileReader : StoreFileReader +class InMemoryStoreFileReader : StoreFileReader { readonly int _length;