Various fixes regarding special tokens (embeddings & anti-prompt)

LLama.Examples/Examples/BatchedExecutorBoolQ.cs

@@ -178,7 +178,7 @@ public ConversationRunner(BatchedExecutor executor, string sys, string question,
 
             // Prompt
             _conversation = executor.Create();
-            _conversation.Prompt(_executor.Context.Tokenize(templatedQuestion));
+            _conversation.Prompt(_executor.Context.Tokenize(templatedQuestion, special: true));
 
             Question = question;
             Answer = answer;

LLama/LLamaEmbedder.cs

@@ -64,7 +64,7 @@ public async Task<IReadOnlyList<float[]>> GetEmbeddings(string input, Cancellati
     private async Task<(IReadOnlyList<float[]> Embeddings, int Tokens)> GetEmbeddingsWithTokenCount(string input, CancellationToken cancellationToken = default)
     {
         // Add all of the tokens to the batch
-        var tokens = Context.Tokenize(input);
+        var tokens = Context.Tokenize(input, special: true);
         var batch = new LLamaBatch();
         for (var i = 0; i < tokens.Length; i++)
             batch.Add(tokens[i], i, LLamaSeqId.Zero, true);

LLama/LLamaInteractExecutor.cs

@@ -315,7 +315,7 @@ protected override async Task InferInternal(IInferenceParams inferenceParams, In
                     id = Context.NativeHandle.ModelHandle.Vocab.Newline!.Value;
                     if (args.Antiprompts is not null && args.Antiprompts.Count > 0)
                     {
-                        var first_antiprompt = Context.Tokenize(args.Antiprompts[0], false);
+                        var first_antiprompt = Context.Tokenize(args.Antiprompts[0], false, true);
                         _embed_inps.AddRange(first_antiprompt);
                     }
                 }