From 143fecfd3685cb4cf72683086c26a3f5c866ca97 Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Wed, 3 Apr 2024 16:59:26 +0100 Subject: [PATCH 1/8] - Added `NativeLogConfig` which allows overriding the llama.cpp log callback - Delaying binding of this into llama.cpp until after `NativeLibraryConfig` has loaded --- LLama.Examples/Program.cs | 9 +++- LLama/Native/NativeApi.Load.cs | 9 ++++ LLama/Native/NativeApi.cs | 14 ++--- LLama/Native/NativeLibraryConfig.cs | 69 ++++++++++++++++++------ LLama/Native/NativeLogConfig.cs | 84 +++++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 27 deletions(-) create mode 100644 LLama/Native/NativeLogConfig.cs diff --git a/LLama.Examples/Program.cs b/LLama.Examples/Program.cs index f8c7ba608..610dc1c3a 100644 --- a/LLama.Examples/Program.cs +++ b/LLama.Examples/Program.cs @@ -20,7 +20,14 @@ __ __ ____ __ NativeLibraryConfig .Instance .WithCuda() - .WithLogs(LLamaLogLevel.Info); + .WithLogs(LLamaLogLevel.Debug) + .WithLogCallback((level, message) => + { + var bg = Console.BackgroundColor; + Console.BackgroundColor = ConsoleColor.Magenta; + Console.WriteLine($"[{level}]: {message}"); + Console.BackgroundColor = bg; + }); // Calling this method forces loading to occur now. NativeApi.llama_empty_call(); diff --git a/LLama/Native/NativeApi.Load.cs b/LLama/Native/NativeApi.Load.cs index b5b3a5308..410ecfe0f 100644 --- a/LLama/Native/NativeApi.Load.cs +++ b/LLama/Native/NativeApi.Load.cs @@ -36,6 +36,15 @@ static NativeApi() // Init llama.cpp backend llama_backend_init(); + + // Set flag to indicate that this has been done. No native library config can be done after this point. + NativeLibraryConfig.LibraryHasLoaded = true; + + // Now that the "loaded" flag is set, configure logging in llama.cpp + if (NativeLibraryConfig.Instance.LogCallback != null) + NativeLogConfig.llama_log_set(NativeLibraryConfig.Instance.LogCallback); + if (NativeLibraryConfig.Instance.LoggerCallback != null) + NativeLogConfig.llama_log_set(NativeLibraryConfig.Instance.LoggerCallback); } #if NET5_0_OR_GREATER diff --git a/LLama/Native/NativeApi.cs b/LLama/Native/NativeApi.cs index d46d48a20..41c1809e0 100644 --- a/LLama/Native/NativeApi.cs +++ b/LLama/Native/NativeApi.cs @@ -5,13 +5,6 @@ namespace LLama.Native { - /// - /// Callback from llama.cpp with log messages - /// - /// - /// - public delegate void LLamaLogCallback(LLamaLogLevel level, string message); - /// /// Direct translation of the llama.cpp API /// @@ -364,8 +357,11 @@ public static int llama_token_to_piece(SafeLlamaModelHandle model, LLamaToken ll /// Register a callback to receive llama log messages /// /// - [DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)] - public static extern void llama_log_set(LLamaLogCallback logCallback); + [Obsolete("Use `NativeLogConfig.llama_log_set` instead")] + public static void llama_log_set(NativeLogConfig.LLamaLogCallback logCallback) + { + NativeLogConfig.llama_log_set(logCallback); + } /// /// Clear the KV cache diff --git a/LLama/Native/NativeLibraryConfig.cs b/LLama/Native/NativeLibraryConfig.cs index c08749ba9..40d6d4e65 100644 --- a/LLama/Native/NativeLibraryConfig.cs +++ b/LLama/Native/NativeLibraryConfig.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Microsoft.Extensions.Logging; namespace LLama.Native { @@ -9,18 +10,8 @@ namespace LLama.Native /// Allows configuration of the native llama.cpp libraries to load and use. /// All configuration must be done before using **any** other LLamaSharp methods! /// - public sealed class NativeLibraryConfig + public sealed partial class NativeLibraryConfig { - /// - /// Get the config instance - /// - public static NativeLibraryConfig Instance { get; } = new(); - - /// - /// Check if the native library has already been loaded. Configuration cannot be modified if this is true. - /// - public static bool LibraryHasLoaded { get; internal set; } = false; - private string? _libraryPath; private string? _libraryPathLLava; @@ -36,12 +27,6 @@ public sealed class NativeLibraryConfig /// private readonly List _searchDirectories = new List(); - private static void ThrowIfLoaded() - { - if (LibraryHasLoaded) - throw new InvalidOperationException("NativeLibraryConfig must be configured before using **any** other LLamaSharp methods!"); - } - #region configurators /// /// Load a specified native library as backend for LLamaSharp. @@ -309,6 +294,56 @@ public override string ToString() } #endif + public sealed partial class NativeLibraryConfig + { + /// + /// Get the config instance + /// + public static NativeLibraryConfig Instance { get; } = new(); + + /// + /// Check if the native library has already been loaded. Configuration cannot be modified if this is true. + /// + public static bool LibraryHasLoaded { get; internal set; } + + internal NativeLogConfig.LLamaLogCallback? LogCallback; + internal ILogger? LoggerCallback; + + private static void ThrowIfLoaded() + { + if (LibraryHasLoaded) + throw new InvalidOperationException("NativeLibraryConfig must be configured before using **any** other LLamaSharp methods!"); + } + + /// + /// Set the log callback that will be used for all llama.cpp log messages + /// + /// + /// + public NativeLibraryConfig WithLogCallback(NativeLogConfig.LLamaLogCallback? callback) + { + ThrowIfLoaded(); + + LogCallback = callback; + LoggerCallback = null; + return this; + } + + /// + /// Set the log callback that will be used for all llama.cpp log messages + /// + /// + /// + public NativeLibraryConfig WithLogCallback(ILogger? logger) + { + ThrowIfLoaded(); + + LogCallback = null; + LoggerCallback = logger; + return this; + } + } + internal enum LibraryName { Llama, diff --git a/LLama/Native/NativeLogConfig.cs b/LLama/Native/NativeLogConfig.cs new file mode 100644 index 000000000..c9a3a751b --- /dev/null +++ b/LLama/Native/NativeLogConfig.cs @@ -0,0 +1,84 @@ +using System; +using System.Runtime.InteropServices; +using Microsoft.Extensions.Logging; + +namespace LLama.Native; + +/// +/// Methods for configuring llama.cpp logging +/// +public class NativeLogConfig +{ + /// + /// Callback from llama.cpp with log messages + /// + /// + /// + public delegate void LLamaLogCallback(LLamaLogLevel level, string message); + + /// + /// Register a callback to receive llama log messages + /// + /// + [DllImport(NativeApi.libraryName, CallingConvention = CallingConvention.Cdecl, EntryPoint = "llama_log_set")] + private static extern void native_llama_log_set(LLamaLogCallback? logCallback); + + /// + /// Register a callback to receive llama log messages + /// + /// +#pragma warning disable IDE1006 // Naming Styles (name imitates llama.cpp) + public static void llama_log_set(LLamaLogCallback? logCallback) +#pragma warning restore IDE1006 // Naming Styles + { + if (NativeLibraryConfig.LibraryHasLoaded) + { + // The library is loaded, just pass the callback directly to llama.cpp + native_llama_log_set(logCallback); + } + else + { + // We can't set the log method yet since that would cause the llama.dll to load. + // Instead configure it to be set when the native library loading is done + NativeLibraryConfig.Instance.WithLogCallback(logCallback); + } + } + + /// + /// Register a callback to receive llama log messages + /// + /// +#pragma warning disable IDE1006 // Naming Styles (name imitates llama.cpp) + public static void llama_log_set(ILogger? logger) +#pragma warning restore IDE1006 // Naming Styles + { + // Clear the logger + if (logger == null) + { + llama_log_set((LLamaLogCallback?)null); + return; + } + + // Bind a function that converts into the correct log level + llama_log_set((level, message) => + { + switch (level) + { + case LLamaLogLevel.Error: + logger.LogError("(llama.cpp): {message}", message); + break; + case LLamaLogLevel.Warning: + logger.LogWarning("(llama.cpp): {message}", message); + break; + case LLamaLogLevel.Info: + logger.LogInformation("(llama.cpp): {message}", message); + break; + case LLamaLogLevel.Debug: + logger.LogDebug("(llama.cpp): {message}", message); + break; + default: + throw new ArgumentOutOfRangeException(nameof(level), level, null); + } + }); + } +} \ No newline at end of file From 0d946450f5a14f97d4688029af5c852c0a9c19cb Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Wed, 3 Apr 2024 18:34:06 +0100 Subject: [PATCH 2/8] Using the log callback to show loading log messages during loading. --- LLama.Examples/Program.cs | 16 +++++++++------- LLama/Native/LLamaLogLevel.cs | 28 +++++++++++++++++++++++----- LLama/Native/NativeApi.Load.cs | 26 ++++++++++++++++++-------- LLama/Native/NativeLogConfig.cs | 23 +++-------------------- 4 files changed, 53 insertions(+), 40 deletions(-) diff --git a/LLama.Examples/Program.cs b/LLama.Examples/Program.cs index 610dc1c3a..280eaccf3 100644 --- a/LLama.Examples/Program.cs +++ b/LLama.Examples/Program.cs @@ -16,17 +16,19 @@ __ __ ____ __ """); -// Configure native library to use +// Configure native library to use. This must be done before any other llama.cpp methods are called! +NativeLibraryConfig + .Instance + .WithCuda(); + +// Configure logging. Change this to `true` to see log messages from llama.cpp +var showLLamaCppLogs = false; NativeLibraryConfig .Instance - .WithCuda() - .WithLogs(LLamaLogLevel.Debug) .WithLogCallback((level, message) => { - var bg = Console.BackgroundColor; - Console.BackgroundColor = ConsoleColor.Magenta; - Console.WriteLine($"[{level}]: {message}"); - Console.BackgroundColor = bg; + if (showLLamaCppLogs) + Console.WriteLine($"[llama {level}]: {message}"); }); // Calling this method forces loading to occur now. diff --git a/LLama/Native/LLamaLogLevel.cs b/LLama/Native/LLamaLogLevel.cs index 39e4545ab..07aca59ed 100644 --- a/LLama/Native/LLamaLogLevel.cs +++ b/LLama/Native/LLamaLogLevel.cs @@ -1,8 +1,11 @@ -namespace LLama.Native -{ - /// - /// Severity level of a log message - /// +using System; +using Microsoft.Extensions.Logging; + +namespace LLama.Native +{ + /// + /// Severity level of a log message + /// public enum LLamaLogLevel { /// @@ -25,4 +28,19 @@ public enum LLamaLogLevel /// Debug = 5, } + + internal static class LLamaLogLevelExtensions + { + public static LogLevel ToLogLevel(this LLamaLogLevel llama) + { + return (llama) switch + { + LLamaLogLevel.Error => LogLevel.Error, + LLamaLogLevel.Warning => LogLevel.Warning, + LLamaLogLevel.Info => LogLevel.Information, + LLamaLogLevel.Debug => LogLevel.Debug, + _ => throw new ArgumentOutOfRangeException(nameof(llama), llama, null) + }; + } + } } diff --git a/LLama/Native/NativeApi.Load.cs b/LLama/Native/NativeApi.Load.cs index 410ecfe0f..c9af6749d 100644 --- a/LLama/Native/NativeApi.Load.cs +++ b/LLama/Native/NativeApi.Load.cs @@ -4,6 +4,7 @@ using System.Runtime.InteropServices; using System.Text.Json; using System.Collections.Generic; +using Microsoft.Extensions.Logging; namespace LLama.Native { @@ -89,41 +90,51 @@ private static void SetDllImportResolver() private static void Log(string message, LLamaLogLevel level) { - if (!enableLogging) + // Log to ILogger + if (NativeLibraryConfig.Instance.LoggerCallback != null) + { + NativeLibraryConfig.Instance.LoggerCallback.Log(level.ToLogLevel(), "{message}", message); return; + } + // Skip other log methods unless enabled + if (!enableLogging) + return; if ((int)level > (int)logLevel) return; + // Log to callback + if (NativeLibraryConfig.Instance.LogCallback != null) + { + NativeLibraryConfig.Instance.LogCallback(level, message); + return; + } + + // Log directly to console var fg = Console.ForegroundColor; var bg = Console.BackgroundColor; try { ConsoleColor color; - string levelPrefix; if (level == LLamaLogLevel.Debug) { color = ConsoleColor.Cyan; - levelPrefix = "[Debug]"; } else if (level == LLamaLogLevel.Info) { color = ConsoleColor.Green; - levelPrefix = "[Info]"; } else if (level == LLamaLogLevel.Error) { color = ConsoleColor.Red; - levelPrefix = "[Error]"; } else { color = ConsoleColor.Yellow; - levelPrefix = "[UNK]"; } Console.ForegroundColor = color; - Console.WriteLine($"{loggingPrefix} {levelPrefix} {message}"); + Console.WriteLine($"[LLamaSharp Native] {level} {message}"); } finally { @@ -464,7 +475,6 @@ string TryFindPath(string filename) internal const string libraryName = "llama"; internal const string llavaLibraryName = "llava_shared"; private const string cudaVersionFile = "version.json"; - private const string loggingPrefix = "[LLamaSharp Native]"; private static bool enableLogging = false; private static LLamaLogLevel logLevel = LLamaLogLevel.Info; } diff --git a/LLama/Native/NativeLogConfig.cs b/LLama/Native/NativeLogConfig.cs index c9a3a751b..fa677a569 100644 --- a/LLama/Native/NativeLogConfig.cs +++ b/LLama/Native/NativeLogConfig.cs @@ -1,11 +1,10 @@ -using System; -using System.Runtime.InteropServices; +using System.Runtime.InteropServices; using Microsoft.Extensions.Logging; namespace LLama.Native; /// -/// Methods for configuring llama.cpp logging +/// Configure llama.cpp logging /// public class NativeLogConfig { @@ -62,23 +61,7 @@ public static void llama_log_set(ILogger? logger) // Bind a function that converts into the correct log level llama_log_set((level, message) => { - switch (level) - { - case LLamaLogLevel.Error: - logger.LogError("(llama.cpp): {message}", message); - break; - case LLamaLogLevel.Warning: - logger.LogWarning("(llama.cpp): {message}", message); - break; - case LLamaLogLevel.Info: - logger.LogInformation("(llama.cpp): {message}", message); - break; - case LLamaLogLevel.Debug: - logger.LogDebug("(llama.cpp): {message}", message); - break; - default: - throw new ArgumentOutOfRangeException(nameof(level), level, null); - } + logger.Log(level.ToLogLevel(), "(llama.cpp): {message}", message); }); } } \ No newline at end of file From 13c919b7016c451310bfaf8178432ee59e341a9a Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Thu, 4 Apr 2024 16:41:23 +0100 Subject: [PATCH 3/8] Registering log callbacks before any calls to llama.cpp except `llama_empty_call`, this is specifically selected to be a method that does nothing and is just there for triggering DLL loading. --- LLama/Native/NativeApi.Load.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/LLama/Native/NativeApi.Load.cs b/LLama/Native/NativeApi.Load.cs index c9af6749d..a164e3e7b 100644 --- a/LLama/Native/NativeApi.Load.cs +++ b/LLama/Native/NativeApi.Load.cs @@ -18,6 +18,9 @@ static NativeApi() // which llama.dll is used. SetDllImportResolver(); + // Set flag to indicate that this point has been passed. No native library config can be done after this point. + NativeLibraryConfig.LibraryHasLoaded = true; + // Immediately make a call which requires loading the llama DLL. This method call // can't fail unless the DLL hasn't been loaded. try @@ -35,17 +38,14 @@ static NativeApi() "to specify it at the very beginning of your code. For more informations about compilation, please refer to LLamaSharp repo on github.\n"); } - // Init llama.cpp backend - llama_backend_init(); - - // Set flag to indicate that this has been done. No native library config can be done after this point. - NativeLibraryConfig.LibraryHasLoaded = true; - - // Now that the "loaded" flag is set, configure logging in llama.cpp + // Now that the "loaded" flag is set configure logging in llama.cpp if (NativeLibraryConfig.Instance.LogCallback != null) NativeLogConfig.llama_log_set(NativeLibraryConfig.Instance.LogCallback); if (NativeLibraryConfig.Instance.LoggerCallback != null) NativeLogConfig.llama_log_set(NativeLibraryConfig.Instance.LoggerCallback); + + // Init llama.cpp backend + llama_backend_init(); } #if NET5_0_OR_GREATER From 243f2038d6580cd6475440c7d60b3ddd6e16a567 Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Thu, 4 Apr 2024 16:49:55 +0100 Subject: [PATCH 4/8] - Removed much of the complexity of logging from `NativeApi.Load`. It always call whatever log callbacks you have registered. - Removed alternative path for `ILogger` in NativeLibraryConfig, instead it redirects to wrapping it in a delegate. --- LLama/Native/NativeApi.Load.cs | 59 +---------------------------- LLama/Native/NativeLibraryConfig.cs | 44 ++------------------- 2 files changed, 5 insertions(+), 98 deletions(-) diff --git a/LLama/Native/NativeApi.Load.cs b/LLama/Native/NativeApi.Load.cs index a164e3e7b..f4058f2a3 100644 --- a/LLama/Native/NativeApi.Load.cs +++ b/LLama/Native/NativeApi.Load.cs @@ -4,7 +4,6 @@ using System.Runtime.InteropServices; using System.Text.Json; using System.Collections.Generic; -using Microsoft.Extensions.Logging; namespace LLama.Native { @@ -41,8 +40,6 @@ static NativeApi() // Now that the "loaded" flag is set configure logging in llama.cpp if (NativeLibraryConfig.Instance.LogCallback != null) NativeLogConfig.llama_log_set(NativeLibraryConfig.Instance.LogCallback); - if (NativeLibraryConfig.Instance.LoggerCallback != null) - NativeLogConfig.llama_log_set(NativeLibraryConfig.Instance.LoggerCallback); // Init llama.cpp backend llama_backend_init(); @@ -90,57 +87,7 @@ private static void SetDllImportResolver() private static void Log(string message, LLamaLogLevel level) { - // Log to ILogger - if (NativeLibraryConfig.Instance.LoggerCallback != null) - { - NativeLibraryConfig.Instance.LoggerCallback.Log(level.ToLogLevel(), "{message}", message); - return; - } - - // Skip other log methods unless enabled - if (!enableLogging) - return; - if ((int)level > (int)logLevel) - return; - - // Log to callback - if (NativeLibraryConfig.Instance.LogCallback != null) - { - NativeLibraryConfig.Instance.LogCallback(level, message); - return; - } - - // Log directly to console - var fg = Console.ForegroundColor; - var bg = Console.BackgroundColor; - try - { - ConsoleColor color; - if (level == LLamaLogLevel.Debug) - { - color = ConsoleColor.Cyan; - } - else if (level == LLamaLogLevel.Info) - { - color = ConsoleColor.Green; - } - else if (level == LLamaLogLevel.Error) - { - color = ConsoleColor.Red; - } - else - { - color = ConsoleColor.Yellow; - } - - Console.ForegroundColor = color; - Console.WriteLine($"[LLamaSharp Native] {level} {message}"); - } - finally - { - Console.ForegroundColor = fg; - Console.BackgroundColor = bg; - } + NativeLibraryConfig.Instance.LogCallback?.Invoke(level, message); } #region CUDA version @@ -382,8 +329,6 @@ private static IntPtr TryLoadLibraries(LibraryName lib) { #if NET6_0_OR_GREATER var configuration = NativeLibraryConfig.CheckAndGatherDescription(lib); - enableLogging = configuration.Logging; - logLevel = configuration.LogLevel; // Set the flag to ensure the NativeLibraryConfig can no longer be modified NativeLibraryConfig.LibraryHasLoaded = true; @@ -475,7 +420,5 @@ string TryFindPath(string filename) internal const string libraryName = "llama"; internal const string llavaLibraryName = "llava_shared"; private const string cudaVersionFile = "version.json"; - private static bool enableLogging = false; - private static LLamaLogLevel logLevel = LLamaLogLevel.Info; } } diff --git a/LLama/Native/NativeLibraryConfig.cs b/LLama/Native/NativeLibraryConfig.cs index 40d6d4e65..ef7cd7c19 100644 --- a/LLama/Native/NativeLibraryConfig.cs +++ b/LLama/Native/NativeLibraryConfig.cs @@ -19,8 +19,6 @@ public sealed partial class NativeLibraryConfig private AvxLevel _avxLevel; private bool _allowFallback = true; private bool _skipCheck = false; - private bool _logging = false; - private LLamaLogLevel _logLevel = LLamaLogLevel.Info; /// /// search directory -> priority level, 0 is the lowest. @@ -102,35 +100,6 @@ public NativeLibraryConfig SkipCheck(bool enable = true) return this; } - /// - /// Whether to output the logs to console when loading the native library with your configuration. - /// - /// - /// - /// Thrown if `LibraryHasLoaded` is true. - public NativeLibraryConfig WithLogs(bool enable) - { - ThrowIfLoaded(); - - _logging = enable; - return this; - } - - /// - /// Enable console logging with the specified log logLevel. - /// - /// - /// - /// Thrown if `LibraryHasLoaded` is true. - public NativeLibraryConfig WithLogs(LLamaLogLevel logLevel = LLamaLogLevel.Info) - { - ThrowIfLoaded(); - - _logging = true; - _logLevel = logLevel; - return this; - } - /// /// Add self-defined search directories. Note that the file stucture of the added /// directories must be the same as the default directory. Besides, the directory @@ -181,8 +150,6 @@ internal static Description CheckAndGatherDescription(LibraryName library) Instance._avxLevel, Instance._allowFallback, Instance._skipCheck, - Instance._logging, - Instance._logLevel, Instance._searchDirectories.Concat(new[] { "./" }).ToArray() ); } @@ -264,7 +231,7 @@ public enum AvxLevel Avx512, } - internal record Description(string? Path, LibraryName Library, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, bool Logging, LLamaLogLevel LogLevel, string[] SearchDirectories) + internal record Description(string? Path, LibraryName Library, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, string[] SearchDirectories) { public override string ToString() { @@ -286,8 +253,6 @@ public override string ToString() $"- PreferredAvxLevel: {avxLevelString}\n" + $"- AllowFallback: {AllowFallback}\n" + $"- SkipCheck: {SkipCheck}\n" + - $"- Logging: {Logging}\n" + - $"- LogLevel: {LogLevel}\n" + $"- SearchDirectories and Priorities: {searchDirectoriesString}"; } } @@ -307,7 +272,6 @@ public sealed partial class NativeLibraryConfig public static bool LibraryHasLoaded { get; internal set; } internal NativeLogConfig.LLamaLogCallback? LogCallback; - internal ILogger? LoggerCallback; private static void ThrowIfLoaded() { @@ -325,7 +289,6 @@ public NativeLibraryConfig WithLogCallback(NativeLogConfig.LLamaLogCallback? cal ThrowIfLoaded(); LogCallback = callback; - LoggerCallback = null; return this; } @@ -338,8 +301,9 @@ public NativeLibraryConfig WithLogCallback(ILogger? logger) { ThrowIfLoaded(); - LogCallback = null; - LoggerCallback = logger; + // Redirect to llama_log_set. This will wrap the logger in a delegate and bind that as the log callback instead. + NativeLogConfig.llama_log_set(logger); + return this; } } From e12cc8455942479057748c59965deaab76d762ae Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Thu, 4 Apr 2024 16:59:08 +0100 Subject: [PATCH 5/8] Saving a GC handle to keep the log callback alive --- LLama/GlobalSuppressions.cs | 2 ++ LLama/Native/NativeLogConfig.cs | 13 ++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/LLama/GlobalSuppressions.cs b/LLama/GlobalSuppressions.cs index 2053bc259..4122be700 100644 --- a/LLama/GlobalSuppressions.cs +++ b/LLama/GlobalSuppressions.cs @@ -8,3 +8,5 @@ [assembly: SuppressMessage("Interoperability", "CA1401:P/Invokes should not be visible", Justification = "LLamaSharp intentionally exports the native llama.cpp API")] [assembly: SuppressMessage("Style", "IDE0070:Use 'System.HashCode'", Justification = "Not compatible with netstandard2.0")] + +[assembly: SuppressMessage("Interoperability", "SYSLIB1054:Use 'LibraryImportAttribute' instead of 'DllImportAttribute' to generate P/Invoke marshalling code at compile time", Justification = "Not compatible with netstandard2.0")] diff --git a/LLama/Native/NativeLogConfig.cs b/LLama/Native/NativeLogConfig.cs index fa677a569..dcac94c81 100644 --- a/LLama/Native/NativeLogConfig.cs +++ b/LLama/Native/NativeLogConfig.cs @@ -6,7 +6,7 @@ namespace LLama.Native; /// /// Configure llama.cpp logging /// -public class NativeLogConfig +public static class NativeLogConfig { /// /// Callback from llama.cpp with log messages @@ -22,6 +22,11 @@ public class NativeLogConfig [DllImport(NativeApi.libraryName, CallingConvention = CallingConvention.Cdecl, EntryPoint = "llama_log_set")] private static extern void native_llama_log_set(LLamaLogCallback? logCallback); + /// + /// A GC handle for the current log callback to ensure the callback is not collected + /// + private static GCHandle? _currentLogCallbackHandle; + /// /// Register a callback to receive llama log messages /// @@ -34,6 +39,12 @@ public static void llama_log_set(LLamaLogCallback? logCallback) { // The library is loaded, just pass the callback directly to llama.cpp native_llama_log_set(logCallback); + + // Save a GC handle, to ensure the callback is not collected + _currentLogCallbackHandle?.Free(); + _currentLogCallbackHandle = null; + if (logCallback != null) + _currentLogCallbackHandle = GCHandle.Alloc(logCallback); } else { From dc75aace98c9f175a483e7d0b0428c8e72f05a14 Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Thu, 4 Apr 2024 18:49:05 +0100 Subject: [PATCH 6/8] Removed prefix, logger should already do that. --- LLama/Native/NativeLogConfig.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LLama/Native/NativeLogConfig.cs b/LLama/Native/NativeLogConfig.cs index dcac94c81..ae85ed7cc 100644 --- a/LLama/Native/NativeLogConfig.cs +++ b/LLama/Native/NativeLogConfig.cs @@ -72,7 +72,7 @@ public static void llama_log_set(ILogger? logger) // Bind a function that converts into the correct log level llama_log_set((level, message) => { - logger.Log(level.ToLogLevel(), "(llama.cpp): {message}", message); + logger.Log(level.ToLogLevel(), "{message}", message); }); } } \ No newline at end of file From 6e3ef9e0b9a5ca6daf0b2e30c65b96fa9eca969b Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Thu, 4 Apr 2024 20:07:16 +0100 Subject: [PATCH 7/8] Buffering up messages until a newline is encountered before passing log message to ILogger. --- LLama.Examples/Program.cs | 2 +- LLama/Native/NativeLogConfig.cs | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/LLama.Examples/Program.cs b/LLama.Examples/Program.cs index 280eaccf3..b24ef406b 100644 --- a/LLama.Examples/Program.cs +++ b/LLama.Examples/Program.cs @@ -28,7 +28,7 @@ __ __ ____ __ .WithLogCallback((level, message) => { if (showLLamaCppLogs) - Console.WriteLine($"[llama {level}]: {message}"); + Console.WriteLine($"[llama {level}]: {message.TrimEnd('\n')}"); }); // Calling this method forces loading to occur now. diff --git a/LLama/Native/NativeLogConfig.cs b/LLama/Native/NativeLogConfig.cs index ae85ed7cc..1e0e10a1b 100644 --- a/LLama/Native/NativeLogConfig.cs +++ b/LLama/Native/NativeLogConfig.cs @@ -1,4 +1,5 @@ using System.Runtime.InteropServices; +using System.Text; using Microsoft.Extensions.Logging; namespace LLama.Native; @@ -69,10 +70,24 @@ public static void llama_log_set(ILogger? logger) return; } - // Bind a function that converts into the correct log level + var builder = new StringBuilder(); + + // Bind a function that combines messages until a newline is encountered, then logs it all as one message llama_log_set((level, message) => { - logger.Log(level.ToLogLevel(), "{message}", message); + lock (builder) + { + builder.Append(message); + + if (!message.EndsWith("\n")) + return; + + // Remove the newline from the end + builder.Remove(builder.Length - 1, 1); + + logger.Log(level.ToLogLevel(), "{message}", builder.ToString()); + builder.Clear(); + } }); } } \ No newline at end of file From e0fd556f41e4411968f51a4a5466c05daf5b44c2 Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Fri, 5 Apr 2024 15:20:15 +0100 Subject: [PATCH 8/8] - Added trailing `\n` to log messages from loading. - Using `ThreadLocal` to ensure messages from separate threads don't get mixed together. --- LLama/Native/NativeApi.Load.cs | 3 +++ LLama/Native/NativeLogConfig.cs | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/LLama/Native/NativeApi.Load.cs b/LLama/Native/NativeApi.Load.cs index f4058f2a3..4b4beea2e 100644 --- a/LLama/Native/NativeApi.Load.cs +++ b/LLama/Native/NativeApi.Load.cs @@ -87,6 +87,9 @@ private static void SetDllImportResolver() private static void Log(string message, LLamaLogLevel level) { + if (!message.EndsWith("\n")) + message += "\n"; + NativeLibraryConfig.Instance.LogCallback?.Invoke(level, message); } diff --git a/LLama/Native/NativeLogConfig.cs b/LLama/Native/NativeLogConfig.cs index 1e0e10a1b..ebcd23d47 100644 --- a/LLama/Native/NativeLogConfig.cs +++ b/LLama/Native/NativeLogConfig.cs @@ -1,5 +1,6 @@ using System.Runtime.InteropServices; using System.Text; +using System.Threading; using Microsoft.Extensions.Logging; namespace LLama.Native; @@ -70,24 +71,23 @@ public static void llama_log_set(ILogger? logger) return; } - var builder = new StringBuilder(); + var builderThread = new ThreadLocal(() => new StringBuilder()); // Bind a function that combines messages until a newline is encountered, then logs it all as one message llama_log_set((level, message) => { - lock (builder) - { - builder.Append(message); + var builder = builderThread.Value!; - if (!message.EndsWith("\n")) - return; + builder.Append(message); - // Remove the newline from the end - builder.Remove(builder.Length - 1, 1); + if (!message.EndsWith("\n")) + return; - logger.Log(level.ToLogLevel(), "{message}", builder.ToString()); - builder.Clear(); - } + // Remove the newline from the end + builder.Remove(builder.Length - 1, 1); + + logger.Log(level.ToLogLevel(), "{message}", builder.ToString()); + builder.Clear(); }); } } \ No newline at end of file