From 7fc85054bfb5d622a669f22c8f96e995085df43d Mon Sep 17 00:00:00 2001 From: Michael Coppola Date: Thu, 18 Jul 2024 18:00:45 -0400 Subject: [PATCH 1/3] llama : Added support for Tekken pre-tokenizer (#8577) Removed uneeded `vocab.tokenizer_clean_spaces` assignment --- convert_hf_to_gguf.py | 3 +++ convert_hf_to_gguf_update.py | 1 + include/llama.h | 1 + src/llama.cpp | 12 ++++++++++++ 4 files changed, 17 insertions(+) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 769d49a8b6f..af225fb23b7 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -593,6 +593,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "b53802fb28e26d645c3a310b34bfe07da813026ec7c7716883404d5e0f8b1901": # ref: https://huggingface.co/core42/jais-13b res = "jais" + if chkhsh == "aa78fe8b04bc622b077520b1fb3d3a5c6f7a53dd375e2361e62599be3cf58de1": + # ref: https://huggingface.co/mistralai/Mistral-Nemo-Base-2407 + res = "tekken" if res is None: logger.warning("\n") diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index e4165ae2d97..29942333704 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -91,6 +91,7 @@ class TOKENIZER_TYPE(IntEnum): {"name": "gemma-2", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/google/gemma-2-9b", }, {"name": "jais", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/core42/jais-13b", }, {"name": "t5", "tokt": TOKENIZER_TYPE.UGM, "repo": "https://huggingface.co/google-t5/t5-small", }, + {"name": "tekken", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mistralai/Mistral-Nemo-Base-2407", }, ] diff --git a/include/llama.h b/include/llama.h index c0fb53060ea..36ffec32a25 100644 --- a/include/llama.h +++ b/include/llama.h @@ -92,6 +92,7 @@ extern "C" { LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17, LLAMA_VOCAB_PRE_TYPE_VIKING = 18, LLAMA_VOCAB_PRE_TYPE_JAIS = 19, + LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20, }; // note: these values should be synchronized with ggml_rope diff --git a/src/llama.cpp b/src/llama.cpp index 20e85b3ebe5..fd22d61d7e8 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -5517,6 +5517,11 @@ static void llm_load_vocab( tokenizer_pre == "viking") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_VIKING; vocab.tokenizer_clean_spaces = false; + } else if ( + tokenizer_pre == "tekken") { + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_TEKKEN; + vocab.tokenizer_ignore_merges = true; + vocab.tokenizer_add_bos = true; } else if ( tokenizer_pre == "jais") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_JAIS; @@ -15581,6 +15586,13 @@ struct llm_tokenizer_bpe { "\\p{N}", }; break; + case LLAMA_VOCAB_PRE_TYPE_TEKKEN: + // original regex from tokenizer.json + // "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" + regex_exprs = { + "[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))*((?=[\\p{L}])([^A-Z]))+|[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))+((?=[\\p{L}])([^A-Z]))*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + }; + break; default: // default regex for BPE tokenization pre-processing regex_exprs = { From 447c08092dac9ced3058e5fcf1282c1ad8153e27 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 19 Jul 2024 13:21:38 +0300 Subject: [PATCH 2/3] llama : fix order of pre-tokenizers --- src/llama.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index fd22d61d7e8..8d2e56080dd 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -5517,14 +5517,14 @@ static void llm_load_vocab( tokenizer_pre == "viking") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_VIKING; vocab.tokenizer_clean_spaces = false; + } else if ( + tokenizer_pre == "jais") { + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_JAIS; } else if ( tokenizer_pre == "tekken") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_TEKKEN; vocab.tokenizer_ignore_merges = true; vocab.tokenizer_add_bos = true; - } else if ( - tokenizer_pre == "jais") { - vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_JAIS; } else { throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str())); } From dd5a0bfffc9983258cc8827da4a476a907b853fe Mon Sep 17 00:00:00 2001 From: Michael Coppola Date: Fri, 19 Jul 2024 19:09:08 -0400 Subject: [PATCH 3/3] * Tekken pre-tokenizer no longer uses clean_up_tokenization_spaces * Updated chkhsh for Tekken tokenizer --- convert_hf_to_gguf.py | 2 +- src/llama.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index af225fb23b7..5ed9e9a0cb6 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -593,7 +593,7 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "b53802fb28e26d645c3a310b34bfe07da813026ec7c7716883404d5e0f8b1901": # ref: https://huggingface.co/core42/jais-13b res = "jais" - if chkhsh == "aa78fe8b04bc622b077520b1fb3d3a5c6f7a53dd375e2361e62599be3cf58de1": + if chkhsh == "63b97e4253352e6f357cc59ea5b583e3a680eaeaf2632188c2b952de2588485e": # ref: https://huggingface.co/mistralai/Mistral-Nemo-Base-2407 res = "tekken" diff --git a/src/llama.cpp b/src/llama.cpp index 64cc491496c..dac28365b7d 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -5523,6 +5523,7 @@ static void llm_load_vocab( } else if ( tokenizer_pre == "tekken") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_TEKKEN; + vocab.tokenizer_clean_spaces = false; vocab.tokenizer_ignore_merges = true; vocab.tokenizer_add_bos = true; } else {