From 39704ec468e35c4d47e8f08ee8af0c10bb77a171 Mon Sep 17 00:00:00 2001
From: manishshettym <manishs@berkeley.edu>
Date: Sun, 17 Sep 2023 20:45:58 -0700
Subject: [PATCH 1/3] bump llama_tokenize API to latest specs

---
 llama_cpp/llama_cpp.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py
index f7e562d9f..86c0ac0db 100644
--- a/llama_cpp/llama_cpp.py
+++ b/llama_cpp/llama_cpp.py
@@ -956,14 +956,22 @@ def llama_token_nl(ctx: llama_context_p) -> llama_token:
 def llama_tokenize(
     ctx: llama_context_p,
     text: bytes,
+    text_len: Union[c_int, int],
     tokens,  # type: Array[llama_token]
     n_max_tokens: Union[c_int, int],
     add_bos: Union[c_bool, int],
 ) -> int:
-    return _lib.llama_tokenize(ctx, text, tokens, n_max_tokens, add_bos)
+    return _lib.llama_tokenize(ctx, text, text_len, tokens, n_max_tokens, add_bos)
 
 
-_lib.llama_tokenize.argtypes = [llama_context_p, c_char_p, llama_token_p, c_int, c_bool]
+_lib.llama_tokenize.argtypes = [
+    llama_context_p,
+    c_char_p,
+    c_int,
+    llama_token_p,
+    c_int,
+    c_bool,
+]
 _lib.llama_tokenize.restype = c_int
 
 
@@ -976,16 +984,18 @@ def llama_tokenize(
 def llama_tokenize_with_model(
     model: llama_model_p,
     text: bytes,
+    text_len: Union[c_int, int],
     tokens,  # type: Array[llama_token]
     n_max_tokens: Union[c_int, int],
     add_bos: Union[c_bool, bool],
 ) -> int:
-    return _lib.llama_tokenize_with_model(model, text, tokens, n_max_tokens, add_bos)
+    return _lib.llama_tokenize_with_model(model, text, text_len, tokens, n_max_tokens, add_bos)
 
 
 _lib.llama_tokenize_with_model.argtypes = [
     llama_model_p,
     c_char_p,
+    c_int,
     llama_token_p,
     c_int,
     c_bool,

From 6bb3522f08330ac537f0b655d68e43b263cc11e1 Mon Sep 17 00:00:00 2001
From: manishshettym <manishs@berkeley.edu>
Date: Sun, 17 Sep 2023 21:23:02 -0700
Subject: [PATCH 2/3] bump high level API

---
 llama_cpp/llama.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py
index 5d093bef9..6956a179a 100644
--- a/llama_cpp/llama.py
+++ b/llama_cpp/llama.py
@@ -430,6 +430,7 @@ def tokenize(self, text: bytes, add_bos: bool = True) -> List[int]:
         n_tokens = llama_cpp.llama_tokenize_with_model(
             self.model,
             text,
+            len(text)
             tokens,
             n_ctx,
             add_bos,
@@ -440,6 +441,7 @@ def tokenize(self, text: bytes, add_bos: bool = True) -> List[int]:
             n_tokens = llama_cpp.llama_tokenize_with_model(
                 self.model,
                 text,
+                len(text),
                 tokens,
                 n_tokens,
                 add_bos,

From 90e274bea28e9db99e43f40b5a577ebfaf99946a Mon Sep 17 00:00:00 2001
From: manishshettym <manishs@berkeley.edu>
Date: Sun, 17 Sep 2023 21:25:10 -0700
Subject: [PATCH 3/3] bug fix

---
 llama_cpp/llama.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py
index 6956a179a..0c0d48fa7 100644
--- a/llama_cpp/llama.py
+++ b/llama_cpp/llama.py
@@ -430,7 +430,7 @@ def tokenize(self, text: bytes, add_bos: bool = True) -> List[int]:
         n_tokens = llama_cpp.llama_tokenize_with_model(
             self.model,
             text,
-            len(text)
+            len(text),
             tokens,
             n_ctx,
             add_bos,