diff --git a/LLama.Unittest/BasicTest.cs b/LLama.Unittest/BasicTest.cs index 84fcaa41e..6bd6f0977 100644 --- a/LLama.Unittest/BasicTest.cs +++ b/LLama.Unittest/BasicTest.cs @@ -29,9 +29,9 @@ public void Dispose() [Fact] public void BasicModelProperties() { - Assert.Equal(32000, _model.VocabCount); - Assert.Equal(4096, _model.ContextSize); - Assert.Equal(4096, _model.EmbeddingSize); + Assert.Equal(128256, _model.VocabCount); + Assert.Equal(131072, _model.ContextSize); + Assert.Equal(2048, _model.EmbeddingSize); } [Fact] @@ -41,36 +41,32 @@ public void AdvancedModelProperties() // tests are switched to use a new model! var expected = new Dictionary { - { "general.name", "LLaMA v2" }, + { "general.name", "Llama 3.2 1B Instruct" }, { "general.architecture", "llama" }, { "general.quantization_version", "2" }, - { "general.file_type", "11" }, + { "general.file_type", "2" }, - { "llama.context_length", "4096" }, - { "llama.rope.dimension_count", "128" }, - { "llama.embedding_length", "4096" }, - { "llama.block_count", "32" }, - { "llama.feed_forward_length", "11008" }, + { "llama.context_length", "131072" }, + { "llama.rope.dimension_count", "64" }, + { "llama.embedding_length", "2048" }, + { "llama.block_count", "16" }, + { "llama.feed_forward_length", "8192" }, { "llama.attention.head_count", "32" }, - { "llama.attention.head_count_kv", "32" }, - { "llama.attention.layer_norm_rms_epsilon", "0.000001" }, + { "llama.attention.head_count_kv", "8" }, + { "llama.attention.layer_norm_rms_epsilon", "0.000010" }, - { "tokenizer.ggml.eos_token_id", "2" }, - { "tokenizer.ggml.model", "llama" }, - { "tokenizer.ggml.bos_token_id", "1" }, - { "tokenizer.ggml.unknown_token_id", "0" }, + { "tokenizer.ggml.eos_token_id", "128009" }, + { "tokenizer.ggml.model", "gpt2" }, + { "tokenizer.ggml.bos_token_id", "128000" }, }; // Print all keys foreach (var (key, value) in _model.Metadata) _testOutputHelper.WriteLine($"{key} = {value}"); - // Check the count is equal - Assert.Equal(expected.Count, _model.Metadata.Count); - // Check every key - foreach (var (key, value) in _model.Metadata) - Assert.Equal(expected[key], value); + foreach (var (key, value) in expected) + Assert.Equal(_model.Metadata[key], value); } } } \ No newline at end of file diff --git a/LLama.Unittest/Constants.cs b/LLama.Unittest/Constants.cs index 4852a335e..7239ae49d 100644 --- a/LLama.Unittest/Constants.cs +++ b/LLama.Unittest/Constants.cs @@ -1,10 +1,10 @@ -using System.Runtime.InteropServices; +using System.Runtime.InteropServices; namespace LLama.Unittest { internal static class Constants { - public static readonly string GenerativeModelPath = "Models/llama-2-7b-chat.Q3_K_S.gguf"; + public static readonly string GenerativeModelPath = "Models/Llama-3.2-1B-Instruct-Q4_0.gguf"; public static readonly string EmbeddingModelPath = "Models/all-MiniLM-L12-v2.Q8_0.gguf"; public static readonly string LLavaModelPath = "Models/llava-v1.6-mistral-7b.Q3_K_XS.gguf"; diff --git a/LLama.Unittest/KernelMemory/ITextTokenizerTests.cs b/LLama.Unittest/KernelMemory/ITextTokenizerTests.cs index 4000525cc..c06a8f0a3 100644 --- a/LLama.Unittest/KernelMemory/ITextTokenizerTests.cs +++ b/LLama.Unittest/KernelMemory/ITextTokenizerTests.cs @@ -39,7 +39,7 @@ public void GetTokens_ShouldReturnListOfTokensForInputString(string? text) var tokens = _generator!.GetTokens(text); var tokensCount = _generator.CountTokens(text); - var expected = " " + text; // the placement of the space corresponding to BOS will vary by model tokenizer + var expected = text; var actual = string.Join("", tokens); _testOutputHelper.WriteLine($"Tokens for '{text}':"); @@ -79,7 +79,7 @@ public void GetTokens_Unicode_ShouldReturnListOfTokensForInputString(string? tex var tokens = _generator!.GetTokens(text); var tokensCount = _generator.CountTokens(text); - var expected = " " + text; // the placement of the space corresponding to BOS will vary by model tokenizer + var expected = text; var actual = string.Join("", tokens); _testOutputHelper.WriteLine($"Tokens for '{text}':"); diff --git a/LLama.Unittest/LLama.Unittest.csproj b/LLama.Unittest/LLama.Unittest.csproj index 47a29fe38..e1913c626 100644 --- a/LLama.Unittest/LLama.Unittest.csproj +++ b/LLama.Unittest/LLama.Unittest.csproj @@ -29,7 +29,7 @@ - + @@ -57,7 +57,7 @@ PreserveNewest - + PreserveNewest diff --git a/LLama.Unittest/LLamaContextTests.cs b/LLama.Unittest/LLamaContextTests.cs index dcec913c5..e9126ede7 100644 --- a/LLama.Unittest/LLamaContextTests.cs +++ b/LLama.Unittest/LLamaContextTests.cs @@ -30,8 +30,8 @@ public void Dispose() public void CheckProperties() { Assert.Equal(128u, _context.ContextSize); - Assert.Equal(4096, _context.EmbeddingSize); - Assert.Equal(32000, _context.VocabCount); + Assert.Equal(2048, _context.EmbeddingSize); + Assert.Equal(128256, _context.VocabCount); } [Fact] @@ -39,7 +39,7 @@ public void Tokenize() { var tokens = _context.Tokenize("The quick brown fox", true); - Assert.Equal(new LLamaToken[] { 1, 450, 4996, 17354, 1701, 29916 }, tokens); + Assert.Equal(new LLamaToken[] { 128000, 791, 4062, 14198, 39935 }, tokens); } [Fact] @@ -47,7 +47,7 @@ public void TokenizeNewline() { var tokens = _context.Tokenize("\n", false, false); - Assert.Equal(new LLamaToken[] { 29871, 13 }, tokens); + Assert.Equal(new LLamaToken[] { 198 }, tokens); } [Fact] @@ -78,7 +78,7 @@ public void TokenizeWithoutBOS() { var tokens = _context.Tokenize("The quick brown fox", false); - Assert.Equal(new LLamaToken[] { 450, 4996, 17354, 1701, 29916 }, tokens); + Assert.Equal(new LLamaToken[] { 791, 4062, 14198, 39935 }, tokens); } [Fact] diff --git a/LLama.Unittest/LLamaEmbedderTests.cs b/LLama.Unittest/LLamaEmbedderTests.cs index f70a3c830..28e7427f4 100644 --- a/LLama.Unittest/LLamaEmbedderTests.cs +++ b/LLama.Unittest/LLamaEmbedderTests.cs @@ -60,8 +60,8 @@ private async Task CompareEmbeddings(string modelPath) Assert.All(cat.Zip(embeddings[0].Vector.Span.EuclideanNormalization()), e => Assert.Equal(e.First, e.Second, 0.001)); Assert.All(kitten.Zip(embeddings[1].Vector.Span.EuclideanNormalization()), e => Assert.Equal(e.First, e.Second, 0.001)); Assert.All(spoon.Zip(embeddings[2].Vector.Span.EuclideanNormalization()), e => Assert.Equal(e.First, e.Second, 0.001)); - Assert.True(embeddings.Usage?.InputTokenCount is 19 or 20); - Assert.True(embeddings.Usage?.TotalTokenCount is 19 or 20); + Assert.True(embeddings.Usage?.InputTokenCount is 16 or 19); + Assert.True(embeddings.Usage?.TotalTokenCount is 16 or 19); _testOutputHelper.WriteLine($"Cat = [{string.Join(",", cat.AsMemory().Slice(0, 7).ToArray())}...]"); _testOutputHelper.WriteLine($"Kitten = [{string.Join(",", kitten.AsMemory().Slice(0, 7).ToArray())}...]"); diff --git a/LLama.Unittest/LLavaWeightsTests.cs b/LLama.Unittest/LLavaWeightsTests.cs index 30d41fd90..25a5f996a 100644 --- a/LLama.Unittest/LLavaWeightsTests.cs +++ b/LLama.Unittest/LLavaWeightsTests.cs @@ -1,4 +1,4 @@ -using LLama.Common; +using LLama.Common; using LLama.Native; namespace LLama.Unittest @@ -14,7 +14,7 @@ public sealed class LLavaWeightTests public LLavaWeightTests() { - var @params = new ModelParams(Constants.GenerativeModelPath) + var @params = new ModelParams(Constants.LLavaModelPath) { // Llava models requires big context ContextSize = 4096, diff --git a/LLama.Unittest/Native/SafeLlamaModelHandleTests.cs b/LLama.Unittest/Native/SafeLlamaModelHandleTests.cs index 5211d4f6a..97dd46193 100644 --- a/LLama.Unittest/Native/SafeLlamaModelHandleTests.cs +++ b/LLama.Unittest/Native/SafeLlamaModelHandleTests.cs @@ -1,4 +1,4 @@ -using System.Text; +using System.Text; using LLama.Common; using LLama.Native; using LLama.Extensions; @@ -29,7 +29,7 @@ public void MetadataValByKey_ReturnsCorrectly() var template = _model.NativeHandle.MetadataValueByKey(key); var name = Encoding.UTF8.GetStringFromSpan(template!.Value.Span); - const string expected = "LLaMA v2"; + const string expected = "Llama 3.2 1B Instruct"; Assert.Equal(expected, name); var metadataLookup = _model.Metadata[key]; diff --git a/LLama.Unittest/TemplateTests.cs b/LLama.Unittest/TemplateTests.cs index 32d6fa129..a4475547f 100644 --- a/LLama.Unittest/TemplateTests.cs +++ b/LLama.Unittest/TemplateTests.cs @@ -55,20 +55,14 @@ public void BasicTemplate() Assert.Equal(8, templater.Count); var templateResult = Encoding.UTF8.GetString(dest); - const string expected = "<|im_start|>assistant\nhello<|im_end|>\n" + - "<|im_start|>user\nworld<|im_end|>\n" + - "<|im_start|>assistant\n" + - "111<|im_end|>" + - "\n<|im_start|>user\n" + - "aaa<|im_end|>\n" + - "<|im_start|>assistant\n" + - "222<|im_end|>\n" + - "<|im_start|>user\n" + - "bbb<|im_end|>\n" + - "<|im_start|>assistant\n" + - "333<|im_end|>\n" + - "<|im_start|>user\n" + - "ccc<|im_end|>\n"; + const string expected = "<|start_header_id|>assistant<|end_header_id|>\n\nhello<|eot_id|>" + + "<|start_header_id|>user<|end_header_id|>\n\nworld<|eot_id|>" + + "<|start_header_id|>assistant<|end_header_id|>\n\n111<|eot_id|>" + + "<|start_header_id|>user<|end_header_id|>\n\naaa<|eot_id|>" + + "<|start_header_id|>assistant<|end_header_id|>\n\n222<|eot_id|>" + + "<|start_header_id|>user<|end_header_id|>\n\nbbb<|eot_id|>" + + "<|start_header_id|>assistant<|end_header_id|>\n\n333<|eot_id|>" + + "<|start_header_id|>user<|end_header_id|>\n\nccc<|eot_id|>"; Assert.Equal(expected, templateResult); } @@ -136,21 +130,15 @@ public void BasicTemplateWithAddAssistant() Assert.Equal(8, templater.Count); var templateResult = Encoding.UTF8.GetString(dest); - const string expected = "<|im_start|>assistant\nhello<|im_end|>\n" + - "<|im_start|>user\nworld<|im_end|>\n" + - "<|im_start|>assistant\n" + - "111<|im_end|>" + - "\n<|im_start|>user\n" + - "aaa<|im_end|>\n" + - "<|im_start|>assistant\n" + - "222<|im_end|>\n" + - "<|im_start|>user\n" + - "bbb<|im_end|>\n" + - "<|im_start|>assistant\n" + - "333<|im_end|>\n" + - "<|im_start|>user\n" + - "ccc<|im_end|>\n" + - "<|im_start|>assistant\n"; + const string expected = "<|start_header_id|>assistant<|end_header_id|>\n\nhello<|eot_id|>" + + "<|start_header_id|>user<|end_header_id|>\n\nworld<|eot_id|>" + + "<|start_header_id|>assistant<|end_header_id|>\n\n111<|eot_id|>" + + "<|start_header_id|>user<|end_header_id|>\n\naaa<|eot_id|>" + + "<|start_header_id|>assistant<|end_header_id|>\n\n222<|eot_id|>" + + "<|start_header_id|>user<|end_header_id|>\n\nbbb<|eot_id|>" + + "<|start_header_id|>assistant<|end_header_id|>\n\n333<|eot_id|>" + + "<|start_header_id|>user<|end_header_id|>\n\nccc<|eot_id|>" + + "<|start_header_id|>assistant<|end_header_id|>\n\n"; Assert.Equal(expected, templateResult); } @@ -252,14 +240,14 @@ public void Clear_ResetsTemplateState() var dest = templater.Apply(); var templateResult = Encoding.UTF8.GetString(dest); - const string expectedTemplate = $"<|im_start|>user\n{userData}<|im_end|>\n"; + const string expectedTemplate = $"<|start_header_id|>user<|end_header_id|>\n\n{userData}<|eot_id|>"; Assert.Equal(expectedTemplate, templateResult); } [Fact] public void EndOTurnToken_ReturnsExpected() { - Assert.Null(_model.Tokens.EndOfTurnToken); + Assert.Equal("<|eot_id|>", _model.Tokens.EndOfTurnToken); } [Fact] @@ -272,7 +260,7 @@ public void EndOSpeechToken_ReturnsExpected() var eosStr = ConvertTokenToString(_model.Tokens.EOS!.Value); _output.WriteLine(eosStr ?? "null"); - Assert.Equal("", _model.Tokens.EndOfSpeechToken); + Assert.Equal("<|eot_id|>", _model.Tokens.EndOfSpeechToken); } private string? ConvertTokenToString(LLamaToken token) diff --git a/LLama.Unittest/Transformers/PromptTemplateTransformerTests.cs b/LLama.Unittest/Transformers/PromptTemplateTransformerTests.cs index 9b1255f9b..2084298e6 100644 --- a/LLama.Unittest/Transformers/PromptTemplateTransformerTests.cs +++ b/LLama.Unittest/Transformers/PromptTemplateTransformerTests.cs @@ -1,4 +1,4 @@ -using LLama.Common; +using LLama.Common; using LLama.Transformers; namespace LLama.Unittest.Transformers; @@ -28,9 +28,7 @@ public void HistoryToText_EncodesCorrectly() Messages = [new ChatHistory.Message(AuthorRole.User, userData)] }); - const string expected = "<|im_start|>user\n" + - $"{userData}<|im_end|>\n" + - "<|im_start|>assistant\n"; + const string expected = $"<|start_header_id|>user<|end_header_id|>\n\n{userData}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"; Assert.Equal(expected, template); } @@ -62,21 +60,15 @@ public void ToModelPrompt_FormatsCorrectly() // Call once with empty array to discover length var templateResult = PromptTemplateTransformer.ToModelPrompt(templater); - const string expected = "<|im_start|>assistant\nhello<|im_end|>\n" + - "<|im_start|>user\nworld<|im_end|>\n" + - "<|im_start|>assistant\n" + - "111<|im_end|>" + - "\n<|im_start|>user\n" + - "aaa<|im_end|>\n" + - "<|im_start|>assistant\n" + - "222<|im_end|>\n" + - "<|im_start|>user\n" + - "bbb<|im_end|>\n" + - "<|im_start|>assistant\n" + - "333<|im_end|>\n" + - "<|im_start|>user\n" + - "ccc<|im_end|>\n" + - "<|im_start|>assistant\n"; + const string expected = "<|start_header_id|>assistant<|end_header_id|>\n\nhello<|eot_id|>" + + "<|start_header_id|>user<|end_header_id|>\n\nworld<|eot_id|>" + + "<|start_header_id|>assistant<|end_header_id|>\n\n111<|eot_id|>" + + "<|start_header_id|>user<|end_header_id|>\n\naaa<|eot_id|>" + + "<|start_header_id|>assistant<|end_header_id|>\n\n222<|eot_id|>" + + "<|start_header_id|>user<|end_header_id|>\n\nbbb<|eot_id|>" + + "<|start_header_id|>assistant<|end_header_id|>\n\n333<|eot_id|>" + + "<|start_header_id|>user<|end_header_id|>\n\nccc<|eot_id|>" + + "<|start_header_id|>assistant<|end_header_id|>\n\n"; Assert.Equal(expected, templateResult); }