From eb3d2b034e3b580ad297e70efeb6d28df946ab53 Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Tue, 18 Feb 2025 01:36:22 +0000 Subject: [PATCH] Introduced another small model for testing: SmolLM-360M. Only 2 tests are still using the larger model, the `ITextTokenizerTests` crash with this model. --- LLama.Unittest/BasicTest.cs | 30 +++----- LLama.Unittest/Constants.cs | 1 + .../KernelMemory/ITextTokenizerTests.cs | 8 +-- LLama.Unittest/LLama.Unittest.csproj | 6 ++ LLama.Unittest/LLamaContextTests.cs | 18 ++--- .../LLamaContextWithCustomLoggerTests.cs | 6 +- LLama.Unittest/LLamaEmbedderTests.cs | 8 +-- LLama.Unittest/MemoryDisposalTests.cs | 4 +- .../Native/SafeLlamaModelHandleTests.cs | 8 +-- LLama.Unittest/SamplingTests.cs | 2 +- LLama.Unittest/StatelessExecutorTest.cs | 2 +- LLama.Unittest/StreamingTextDecoderTests.cs | 4 +- LLama.Unittest/TemplateTests.cs | 72 ++++++++----------- LLama.Unittest/TokenTests.cs | 4 +- .../PromptTemplateTransformerTests.cs | 38 ++++++---- 15 files changed, 95 insertions(+), 116 deletions(-) diff --git a/LLama.Unittest/BasicTest.cs b/LLama.Unittest/BasicTest.cs index 03582836f..6624f417f 100644 --- a/LLama.Unittest/BasicTest.cs +++ b/LLama.Unittest/BasicTest.cs @@ -13,7 +13,7 @@ public sealed class BasicTest public BasicTest(ITestOutputHelper testOutputHelper) { _testOutputHelper = testOutputHelper; - _params = new ModelParams(Constants.GenerativeModelPath) + _params = new ModelParams(Constants.GenerativeModelPath2) { ContextSize = 128, GpuLayerCount = Constants.CIGpuLayerCount @@ -26,14 +26,6 @@ public void Dispose() _model.Dispose(); } - [Fact] - public void BasicModelProperties() - { - Assert.Equal(128256, _model.Vocab.Count); - Assert.Equal(131072, _model.ContextSize); - Assert.Equal(2048, _model.EmbeddingSize); - } - [Fact] public void AdvancedModelProperties() { @@ -41,23 +33,23 @@ public void AdvancedModelProperties() // tests are switched to use a new model! var expected = new Dictionary { - { "general.name", "Llama 3.2 1B Instruct" }, + { "general.name", "SmolLM 360M" }, { "general.architecture", "llama" }, { "general.quantization_version", "2" }, - { "general.file_type", "2" }, + { "general.file_type", "7" }, - { "llama.context_length", "131072" }, + { "llama.context_length", "2048" }, { "llama.rope.dimension_count", "64" }, - { "llama.embedding_length", "2048" }, - { "llama.block_count", "16" }, - { "llama.feed_forward_length", "8192" }, - { "llama.attention.head_count", "32" }, - { "llama.attention.head_count_kv", "8" }, + { "llama.embedding_length", "960" }, + { "llama.block_count", "32" }, + { "llama.feed_forward_length", "2560" }, + { "llama.attention.head_count", "15" }, + { "llama.attention.head_count_kv", "5" }, { "llama.attention.layer_norm_rms_epsilon", "0.000010" }, - { "tokenizer.ggml.eos_token_id", "128009" }, + { "tokenizer.ggml.eos_token_id", "2" }, { "tokenizer.ggml.model", "gpt2" }, - { "tokenizer.ggml.bos_token_id", "128000" }, + { "tokenizer.ggml.bos_token_id", "1" }, }; // Print all keys diff --git a/LLama.Unittest/Constants.cs b/LLama.Unittest/Constants.cs index 7239ae49d..a30951750 100644 --- a/LLama.Unittest/Constants.cs +++ b/LLama.Unittest/Constants.cs @@ -5,6 +5,7 @@ namespace LLama.Unittest internal static class Constants { public static readonly string GenerativeModelPath = "Models/Llama-3.2-1B-Instruct-Q4_0.gguf"; + public static readonly string GenerativeModelPath2 = "Models/smollm-360m-instruct-add-basics-q8_0.gguf"; public static readonly string EmbeddingModelPath = "Models/all-MiniLM-L12-v2.Q8_0.gguf"; public static readonly string LLavaModelPath = "Models/llava-v1.6-mistral-7b.Q3_K_XS.gguf"; diff --git a/LLama.Unittest/KernelMemory/ITextTokenizerTests.cs b/LLama.Unittest/KernelMemory/ITextTokenizerTests.cs index c06a8f0a3..a12ad04ee 100644 --- a/LLama.Unittest/KernelMemory/ITextTokenizerTests.cs +++ b/LLama.Unittest/KernelMemory/ITextTokenizerTests.cs @@ -14,8 +14,8 @@ public abstract class ITextTokenizerTests protected ITextTokenizer? _generator; #pragma warning restore KMEXP00 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - protected InferenceParams _infParams; - protected LLamaSharpConfig _lsConfig; + protected readonly InferenceParams _infParams; + protected readonly LLamaSharpConfig _lsConfig; public ITextTokenizerTests(ITestOutputHelper testOutputHelper) { @@ -34,7 +34,7 @@ public ITextTokenizerTests(ITestOutputHelper testOutputHelper) [InlineData("...___---")] [InlineData("15 + 6 = 21 && 68 * 75 = 5100")] [InlineData(" \n \r\n \t ")] - public void GetTokens_ShouldReturnListOfTokensForInputString(string? text) + public void GetTokens_ShouldReturnListOfTokensForInputString(string text) { var tokens = _generator!.GetTokens(text); var tokensCount = _generator.CountTokens(text); @@ -74,7 +74,7 @@ public void GetTokens_ShouldReturnListOfTokensForInputString(string? text) [Theory] [InlineData("And a little bit of unicode για να κρατήσουμε τα πράγματα ενδιαφέροντα")] [InlineData("猫坐在垫子上 😀🤨🤐😏")] - public void GetTokens_Unicode_ShouldReturnListOfTokensForInputString(string? text) + public void GetTokens_Unicode_ShouldReturnListOfTokensForInputString(string text) { var tokens = _generator!.GetTokens(text); var tokensCount = _generator.CountTokens(text); diff --git a/LLama.Unittest/LLama.Unittest.csproj b/LLama.Unittest/LLama.Unittest.csproj index 0836a384a..5d7277e7a 100644 --- a/LLama.Unittest/LLama.Unittest.csproj +++ b/LLama.Unittest/LLama.Unittest.csproj @@ -31,6 +31,9 @@ + + + @@ -60,6 +63,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/LLama.Unittest/LLamaContextTests.cs b/LLama.Unittest/LLamaContextTests.cs index e033ffad8..82157a17f 100644 --- a/LLama.Unittest/LLamaContextTests.cs +++ b/LLama.Unittest/LLamaContextTests.cs @@ -11,7 +11,7 @@ public sealed class LLamaContextTests public LLamaContextTests() { - var @params = new ModelParams(Constants.GenerativeModelPath) + var @params = new ModelParams(Constants.GenerativeModelPath2) { ContextSize = 128, GpuLayerCount = Constants.CIGpuLayerCount, @@ -30,16 +30,16 @@ public void Dispose() public void CheckProperties() { Assert.Equal(128u, _context.ContextSize); - Assert.Equal(2048, _context.EmbeddingSize); - Assert.Equal(128256, _context.Vocab.Count); + Assert.Equal(960, _context.EmbeddingSize); + Assert.Equal(49152, _context.Vocab.Count); } [Fact] public void Tokenize() { - var tokens = _context.Tokenize("The quick brown fox", true); + var tokens = _context.Tokenize("The quick brown fox"); - Assert.Equal(new LLamaToken[] { 128000, 791, 4062, 14198, 39935 }, tokens); + Assert.Equal(new LLamaToken[] { 504, 2365, 6354, 16438 }, tokens); } [Fact] @@ -73,14 +73,6 @@ public void TokenizeRoundtripSpecialStrings() } } - [Fact] - public void TokenizeWithoutBOS() - { - var tokens = _context.Tokenize("The quick brown fox", false); - - Assert.Equal(new LLamaToken[] { 791, 4062, 14198, 39935 }, tokens); - } - [Fact] public void TokenizeEmpty() { diff --git a/LLama.Unittest/LLamaContextWithCustomLoggerTests.cs b/LLama.Unittest/LLamaContextWithCustomLoggerTests.cs index bbcc2e063..1d16b0481 100644 --- a/LLama.Unittest/LLamaContextWithCustomLoggerTests.cs +++ b/LLama.Unittest/LLamaContextWithCustomLoggerTests.cs @@ -28,7 +28,7 @@ public void Log( public LLamaContextWithCustomLoggerTests() { - var @params = new ModelParams(Constants.GenerativeModelPath) + var @params = new ModelParams(Constants.GenerativeModelPath2) { ContextSize = 128, GpuLayerCount = Constants.CIGpuLayerCount, @@ -56,8 +56,8 @@ public void Dispose() public void CheckProperties() { Assert.Equal(128u, _context.ContextSize); - Assert.Equal(2048, _context.EmbeddingSize); - Assert.Equal(128256, _context.Vocab.Count); + Assert.Equal(960, _context.EmbeddingSize); + Assert.Equal(49152, _context.Vocab.Count); } } } \ No newline at end of file diff --git a/LLama.Unittest/LLamaEmbedderTests.cs b/LLama.Unittest/LLamaEmbedderTests.cs index 28e7427f4..69eae9aa7 100644 --- a/LLama.Unittest/LLamaEmbedderTests.cs +++ b/LLama.Unittest/LLamaEmbedderTests.cs @@ -60,8 +60,6 @@ private async Task CompareEmbeddings(string modelPath) Assert.All(cat.Zip(embeddings[0].Vector.Span.EuclideanNormalization()), e => Assert.Equal(e.First, e.Second, 0.001)); Assert.All(kitten.Zip(embeddings[1].Vector.Span.EuclideanNormalization()), e => Assert.Equal(e.First, e.Second, 0.001)); Assert.All(spoon.Zip(embeddings[2].Vector.Span.EuclideanNormalization()), e => Assert.Equal(e.First, e.Second, 0.001)); - Assert.True(embeddings.Usage?.InputTokenCount is 16 or 19); - Assert.True(embeddings.Usage?.TotalTokenCount is 16 or 19); _testOutputHelper.WriteLine($"Cat = [{string.Join(",", cat.AsMemory().Slice(0, 7).ToArray())}...]"); _testOutputHelper.WriteLine($"Kitten = [{string.Join(",", kitten.AsMemory().Slice(0, 7).ToArray())}...]"); @@ -84,9 +82,9 @@ public async Task EmbedCompareEmbeddingModel() } [Fact] - public async Task EmbedCompareGenerateModel() + public async Task EmbedCompareGenerativeModel() { - await CompareEmbeddings(Constants.GenerativeModelPath); + await CompareEmbeddings(Constants.GenerativeModelPath2); } private async Task NonPooledEmbeddings(string modelPath) @@ -115,6 +113,6 @@ public async Task EmbeddingModelNonPooledEmbeddings() [Fact] public async Task GenerativeModelNonPooledEmbeddings() { - await NonPooledEmbeddings(Constants.GenerativeModelPath); + await NonPooledEmbeddings(Constants.GenerativeModelPath2); } } \ No newline at end of file diff --git a/LLama.Unittest/MemoryDisposalTests.cs b/LLama.Unittest/MemoryDisposalTests.cs index 082fc5bce..0210f7825 100644 --- a/LLama.Unittest/MemoryDisposalTests.cs +++ b/LLama.Unittest/MemoryDisposalTests.cs @@ -7,7 +7,7 @@ public class MemoryDisposalTests [Fact] public void ModelDisposal() { - var @params = new ModelParams(Constants.GenerativeModelPath) + var @params = new ModelParams(Constants.GenerativeModelPath2) { ContextSize = 2048, GpuLayerCount = 0, @@ -22,7 +22,7 @@ public void ModelDisposal() [Fact] public void ContextDisposal() { - var @params = new ModelParams(Constants.GenerativeModelPath) + var @params = new ModelParams(Constants.GenerativeModelPath2) { ContextSize = 128, GpuLayerCount = 0, diff --git a/LLama.Unittest/Native/SafeLlamaModelHandleTests.cs b/LLama.Unittest/Native/SafeLlamaModelHandleTests.cs index 97dd46193..40e56ca63 100644 --- a/LLama.Unittest/Native/SafeLlamaModelHandleTests.cs +++ b/LLama.Unittest/Native/SafeLlamaModelHandleTests.cs @@ -1,6 +1,5 @@ using System.Text; using LLama.Common; -using LLama.Native; using LLama.Extensions; namespace LLama.Unittest.Native; @@ -8,18 +7,15 @@ namespace LLama.Unittest.Native; public class SafeLlamaModelHandleTests { private readonly LLamaWeights _model; - private readonly SafeLlamaModelHandle TestableHandle; public SafeLlamaModelHandleTests() { - var @params = new ModelParams(Constants.GenerativeModelPath) + var @params = new ModelParams(Constants.GenerativeModelPath2) { ContextSize = 1, GpuLayerCount = Constants.CIGpuLayerCount }; _model = LLamaWeights.LoadFromFile(@params); - - TestableHandle = _model.NativeHandle; } [Fact] @@ -29,7 +25,7 @@ public void MetadataValByKey_ReturnsCorrectly() var template = _model.NativeHandle.MetadataValueByKey(key); var name = Encoding.UTF8.GetStringFromSpan(template!.Value.Span); - const string expected = "Llama 3.2 1B Instruct"; + const string expected = "SmolLM 360M"; Assert.Equal(expected, name); var metadataLookup = _model.Metadata[key]; diff --git a/LLama.Unittest/SamplingTests.cs b/LLama.Unittest/SamplingTests.cs index ac57d28e7..615a7c79e 100644 --- a/LLama.Unittest/SamplingTests.cs +++ b/LLama.Unittest/SamplingTests.cs @@ -22,7 +22,7 @@ public class SamplingTests public SamplingTests(ITestOutputHelper testOutputHelper) { _testOutputHelper = testOutputHelper; - _params = new ModelParams(Constants.GenerativeModelPath) { + _params = new ModelParams(Constants.GenerativeModelPath2) { ContextSize = 200, BatchSize = 200, GpuLayerCount = Constants.CIGpuLayerCount, diff --git a/LLama.Unittest/StatelessExecutorTest.cs b/LLama.Unittest/StatelessExecutorTest.cs index afaf5f0af..d2c646e99 100644 --- a/LLama.Unittest/StatelessExecutorTest.cs +++ b/LLama.Unittest/StatelessExecutorTest.cs @@ -15,7 +15,7 @@ public class StatelessExecutorTest public StatelessExecutorTest(ITestOutputHelper testOutputHelper) { _testOutputHelper = testOutputHelper; - _params = new ModelParams(Constants.GenerativeModelPath) + _params = new ModelParams(Constants.GenerativeModelPath2) { ContextSize = 60, BatchSize = 2, diff --git a/LLama.Unittest/StreamingTextDecoderTests.cs b/LLama.Unittest/StreamingTextDecoderTests.cs index 7291b5d07..a53383207 100644 --- a/LLama.Unittest/StreamingTextDecoderTests.cs +++ b/LLama.Unittest/StreamingTextDecoderTests.cs @@ -1,4 +1,4 @@ -using System.Text; +using System.Text; using LLama.Common; using Xunit.Abstractions; @@ -14,7 +14,7 @@ public class StreamingTextDecoderTests public StreamingTextDecoderTests(ITestOutputHelper testOutputHelper) { _testOutputHelper = testOutputHelper; - _params = new ModelParams(Constants.GenerativeModelPath); + _params = new ModelParams(Constants.GenerativeModelPath2); _model = LLamaWeights.LoadFromFile(_params); } diff --git a/LLama.Unittest/TemplateTests.cs b/LLama.Unittest/TemplateTests.cs index c1ef3122a..7579715a2 100644 --- a/LLama.Unittest/TemplateTests.cs +++ b/LLama.Unittest/TemplateTests.cs @@ -15,7 +15,7 @@ public sealed class TemplateTests public TemplateTests(ITestOutputHelper output) { _output = output; - var @params = new ModelParams(Constants.GenerativeModelPath) + var @params = new ModelParams(Constants.GenerativeModelPath2) { ContextSize = 1, GpuLayerCount = Constants.CIGpuLayerCount @@ -55,14 +55,16 @@ public void BasicTemplate() Assert.Equal(8, templater.Count); var templateResult = Encoding.UTF8.GetString(dest); - const string expected = "<|start_header_id|>assistant<|end_header_id|>\n\nhello<|eot_id|>" - + "<|start_header_id|>user<|end_header_id|>\n\nworld<|eot_id|>" - + "<|start_header_id|>assistant<|end_header_id|>\n\n111<|eot_id|>" - + "<|start_header_id|>user<|end_header_id|>\n\naaa<|eot_id|>" - + "<|start_header_id|>assistant<|end_header_id|>\n\n222<|eot_id|>" - + "<|start_header_id|>user<|end_header_id|>\n\nbbb<|eot_id|>" - + "<|start_header_id|>assistant<|end_header_id|>\n\n333<|eot_id|>" - + "<|start_header_id|>user<|end_header_id|>\n\nccc<|eot_id|>"; + const string expected = "<|im_start|>assistant\nhello<|im_end|>\n" + + "<|im_start|>user\nworld<|im_end|>\n" + + "<|im_start|>assistant\n111<|im_end|>\n" + + "<|im_start|>user\naaa<|im_end|>\n" + + "<|im_start|>assistant\n222<|im_end|>\n" + + "<|im_start|>user\nbbb<|im_end|>\n" + + "<|im_start|>assistant\n" + + "333<|im_end|>\n" + + "<|im_start|>user\n" + + "ccc<|im_end|>\n"; var eq = expected == templateResult; Assert.Equal(expected, templateResult); @@ -131,15 +133,23 @@ public void BasicTemplateWithAddAssistant() Assert.Equal(8, templater.Count); var templateResult = Encoding.UTF8.GetString(dest); - const string expected = "<|start_header_id|>assistant<|end_header_id|>\n\nhello<|eot_id|>" - + "<|start_header_id|>user<|end_header_id|>\n\nworld<|eot_id|>" - + "<|start_header_id|>assistant<|end_header_id|>\n\n111<|eot_id|>" - + "<|start_header_id|>user<|end_header_id|>\n\naaa<|eot_id|>" - + "<|start_header_id|>assistant<|end_header_id|>\n\n222<|eot_id|>" - + "<|start_header_id|>user<|end_header_id|>\n\nbbb<|eot_id|>" - + "<|start_header_id|>assistant<|end_header_id|>\n\n333<|eot_id|>" - + "<|start_header_id|>user<|end_header_id|>\n\nccc<|eot_id|>" - + "<|start_header_id|>assistant<|end_header_id|>\n\n"; + const string expected = "<|im_start|>assistant\n" + + "hello<|im_end|>\n" + + "<|im_start|>user\n" + + "world<|im_end|>\n" + + "<|im_start|>assistant\n" + + "111<|im_end|>\n" + + "<|im_start|>user\n" + + "aaa<|im_end|>\n" + + "<|im_start|>assistant\n" + + "222<|im_end|>\n" + + "<|im_start|>user\n" + + "bbb<|im_end|>\n" + + "<|im_start|>assistant\n" + + "333<|im_end|>\n" + + "<|im_start|>user\n" + + "ccc<|im_end|>\n" + + "<|im_start|>assistant\n"; Assert.Equal(expected, templateResult); } @@ -241,31 +251,7 @@ public void Clear_ResetsTemplateState() var dest = templater.Apply(); var templateResult = Encoding.UTF8.GetString(dest); - const string expectedTemplate = $"<|start_header_id|>user<|end_header_id|>\n\n{userData}<|eot_id|>"; + const string expectedTemplate = $"<|im_start|>user\n{userData}<|im_end|>\n"; Assert.Equal(expectedTemplate, templateResult); } - - private string? ConvertTokenToString(LLamaToken token) - { - _output.WriteLine($"ConvertTokenToString: {token}"); - - const int buffSize = 32; - Span buff = stackalloc byte[buffSize]; - var tokenLength = _model.NativeHandle.TokenToSpan(token, buff, 0, true); - - _output.WriteLine($"tokenLength = {tokenLength}"); - if (tokenLength <= 0) - return null; - - // if the original buffer wasn't large enough, create a new one - _output.WriteLine($"tokenLength = {tokenLength}, buffSize = {buffSize}"); - if (tokenLength > buffSize) - { - buff = stackalloc byte[(int)tokenLength]; - _ = _model.NativeHandle.TokenToSpan(token, buff, 0, true); - } - - var slice = buff.Slice(0, (int)tokenLength); - return Encoding.UTF8.GetStringFromSpan(slice); - } } diff --git a/LLama.Unittest/TokenTests.cs b/LLama.Unittest/TokenTests.cs index 03e3927fe..c159a0759 100644 --- a/LLama.Unittest/TokenTests.cs +++ b/LLama.Unittest/TokenTests.cs @@ -1,4 +1,4 @@ -using System.Text; +using System.Text; using LLama.Common; using LLama.Extensions; @@ -12,7 +12,7 @@ public sealed class TokenTests public TokenTests() { - _params = new ModelParams(Constants.GenerativeModelPath) + _params = new ModelParams(Constants.GenerativeModelPath2) { ContextSize = 2048, GpuLayerCount = Constants.CIGpuLayerCount, diff --git a/LLama.Unittest/Transformers/PromptTemplateTransformerTests.cs b/LLama.Unittest/Transformers/PromptTemplateTransformerTests.cs index 57ae3608c..bf1be5250 100644 --- a/LLama.Unittest/Transformers/PromptTemplateTransformerTests.cs +++ b/LLama.Unittest/Transformers/PromptTemplateTransformerTests.cs @@ -3,22 +3,22 @@ namespace LLama.Unittest.Transformers; -public class PromptTemplateTransformerTests +public class PromptTemplateTransformerTests : IDisposable { private readonly LLamaWeights _model; - private readonly PromptTemplateTransformer TestableTransformer; + private readonly PromptTemplateTransformer _testableTransformer; public PromptTemplateTransformerTests() { - var @params = new ModelParams(Constants.GenerativeModelPath) + var @params = new ModelParams(Constants.GenerativeModelPath2) { ContextSize = 1, GpuLayerCount = Constants.CIGpuLayerCount }; _model = LLamaWeights.LoadFromFile(@params); - TestableTransformer = new PromptTemplateTransformer(_model, true); + _testableTransformer = new PromptTemplateTransformer(_model, true); } public void Dispose() @@ -30,11 +30,11 @@ public void Dispose() public void HistoryToText_EncodesCorrectly() { const string userData = nameof(userData); - var template = TestableTransformer.HistoryToText(new ChatHistory(){ + var template = _testableTransformer.HistoryToText(new ChatHistory(){ Messages = [new ChatHistory.Message(AuthorRole.User, userData)] }); - const string expected = $"<|start_header_id|>user<|end_header_id|>\n\n{userData}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"; + const string expected = $"<|im_start|>user\n{userData}<|im_end|>\n<|im_start|>assistant\n"; Assert.Equal(expected, template); } @@ -66,15 +66,23 @@ public void ToModelPrompt_FormatsCorrectly() // Call once with empty array to discover length var templateResult = PromptTemplateTransformer.ToModelPrompt(templater); - const string expected = "<|start_header_id|>assistant<|end_header_id|>\n\nhello<|eot_id|>" - + "<|start_header_id|>user<|end_header_id|>\n\nworld<|eot_id|>" - + "<|start_header_id|>assistant<|end_header_id|>\n\n111<|eot_id|>" - + "<|start_header_id|>user<|end_header_id|>\n\naaa<|eot_id|>" - + "<|start_header_id|>assistant<|end_header_id|>\n\n222<|eot_id|>" - + "<|start_header_id|>user<|end_header_id|>\n\nbbb<|eot_id|>" - + "<|start_header_id|>assistant<|end_header_id|>\n\n333<|eot_id|>" - + "<|start_header_id|>user<|end_header_id|>\n\nccc<|eot_id|>" - + "<|start_header_id|>assistant<|end_header_id|>\n\n"; + const string expected = "<|im_start|>assistant\n" + + "hello<|im_end|>\n" + + "<|im_start|>user\n" + + "world<|im_end|>\n" + + "<|im_start|>assistant\n" + + "111<|im_end|>\n" + + "<|im_start|>user\n" + + "aaa<|im_end|>\n" + + "<|im_start|>assistant\n" + + "222<|im_end|>\n" + + "<|im_start|>user\n" + + "bbb<|im_end|>\n" + + "<|im_start|>assistant\n" + + "333<|im_end|>\n" + + "<|im_start|>user\n" + + "ccc<|im_end|>\n" + + "<|im_start|>assistant\n"; Assert.Equal(expected, templateResult); }