From d1075f6e081fc0e7b60810591ee69778e4fa29fb Mon Sep 17 00:00:00 2001 From: KerfuffleV2 Date: Sun, 15 Oct 2023 07:25:21 -0600 Subject: [PATCH 1/8] Add validation for special token ids to llama.cpp Small optimization for llama_byte_to_token SPM mode --- llama.cpp | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/llama.cpp b/llama.cpp index 82b7638ae7c..03c73ee7bf3 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2235,15 +2235,32 @@ static void llm_load_vocab( if (vocab.type == LLAMA_VOCAB_TYPE_SPM) { vocab.linefeed_id = llama_byte_to_token(vocab, '\n'); } else { - vocab.linefeed_id = llama_tokenize_internal(vocab, "\u010A", false)[0]; + const std::vector ids = llama_tokenize_internal(vocab, "\u010A", false); + GGML_ASSERT(ids.size() == 1 && "model vocab missing newline token"); + vocab.linefeed_id = ids[0]; } // special tokens - GGUF_GET_KEY(ctx, vocab.special_bos_id, gguf_get_val_u32, GGUF_TYPE_UINT32, false, kv(LLM_KV_TOKENIZER_BOS_ID)); - GGUF_GET_KEY(ctx, vocab.special_eos_id, gguf_get_val_u32, GGUF_TYPE_UINT32, false, kv(LLM_KV_TOKENIZER_EOS_ID)); - GGUF_GET_KEY(ctx, vocab.special_unk_id, gguf_get_val_u32, GGUF_TYPE_UINT32, false, kv(LLM_KV_TOKENIZER_UNK_ID)); - GGUF_GET_KEY(ctx, vocab.special_sep_id, gguf_get_val_u32, GGUF_TYPE_UINT32, false, kv(LLM_KV_TOKENIZER_SEP_ID)); - GGUF_GET_KEY(ctx, vocab.special_pad_id, gguf_get_val_u32, GGUF_TYPE_UINT32, false, kv(LLM_KV_TOKENIZER_PAD_ID)); + { + const std::vector> special_token_types = { + { LLM_KV_TOKENIZER_BOS_ID, &vocab.special_bos_id }, + { LLM_KV_TOKENIZER_EOS_ID, &vocab.special_eos_id }, + { LLM_KV_TOKENIZER_UNK_ID, &vocab.special_unk_id }, + { LLM_KV_TOKENIZER_SEP_ID, &vocab.special_sep_id }, + { LLM_KV_TOKENIZER_PAD_ID, &vocab.special_pad_id }, + }; + for (auto & it : special_token_types ) { + int32_t id = -1; + const std::string kstr = kv(std::get<0>(it)); + + GGUF_GET_KEY(ctx, id, gguf_get_val_u32, GGUF_TYPE_UINT32, false, kstr); + if (id != -1 && (id < 0 || size_t(id) >= vocab.id_to_token.size())) { + LLAMA_LOG_WARN("%s: bad special token value %d for key '%s' -- ignoring\n", __func__, id, kstr.c_str()); + continue; + } + *(std::get<1>(it)) = id; + } + } // build special tokens cache { @@ -6084,11 +6101,10 @@ static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) { } static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) { + const char * hex = "0123456789ABCDEF"; switch (llama_vocab_get_type(vocab)) { case LLAMA_VOCAB_TYPE_SPM: { - char buf[7]; - int result = snprintf(buf, sizeof(buf), "<0x%02X>", ch); - GGML_ASSERT(0 <= result && result < 7); + const char buf[7] = { '<', '0', 'x', hex[ch >> 4], hex[ch & 15], '>', 0 }; return vocab.token_to_id.at(buf); } case LLAMA_VOCAB_TYPE_BPE: { From 14be9d91412477d468f163d5ce096640c40e5059 Mon Sep 17 00:00:00 2001 From: KerfuffleV2 Date: Sun, 15 Oct 2023 08:00:53 -0600 Subject: [PATCH 2/8] Fix BPE newline check, only I could break something so simple --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 03c73ee7bf3..f393a52f1a2 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2236,7 +2236,7 @@ static void llm_load_vocab( vocab.linefeed_id = llama_byte_to_token(vocab, '\n'); } else { const std::vector ids = llama_tokenize_internal(vocab, "\u010A", false); - GGML_ASSERT(ids.size() == 1 && "model vocab missing newline token"); + GGML_ASSERT(ids.empty() && "model vocab missing newline token"); vocab.linefeed_id = ids[0]; } From 32383bbd1c5fd7d345ff820d8c4816228f077289 Mon Sep 17 00:00:00 2001 From: KerfuffleV2 Date: Sun, 15 Oct 2023 08:16:18 -0600 Subject: [PATCH 3/8] Killll meeeeee --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index f393a52f1a2..bb5d35af8bc 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2236,7 +2236,7 @@ static void llm_load_vocab( vocab.linefeed_id = llama_byte_to_token(vocab, '\n'); } else { const std::vector ids = llama_tokenize_internal(vocab, "\u010A", false); - GGML_ASSERT(ids.empty() && "model vocab missing newline token"); + GGML_ASSERT(!ids.empty() && "model vocab missing newline token"); vocab.linefeed_id = ids[0]; } From 4079668cda0ab50b82f08dda560e7aef192ff779 Mon Sep 17 00:00:00 2001 From: KerfuffleV2 Date: Sun, 15 Oct 2023 09:48:30 -0600 Subject: [PATCH 4/8] Account for GGUF_KEY_KEY only setting when the key exists --- llama.cpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/llama.cpp b/llama.cpp index bb5d35af8bc..8892691613f 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2242,23 +2242,23 @@ static void llm_load_vocab( // special tokens { - const std::vector> special_token_types = { - { LLM_KV_TOKENIZER_BOS_ID, &vocab.special_bos_id }, - { LLM_KV_TOKENIZER_EOS_ID, &vocab.special_eos_id }, - { LLM_KV_TOKENIZER_UNK_ID, &vocab.special_unk_id }, - { LLM_KV_TOKENIZER_SEP_ID, &vocab.special_sep_id }, - { LLM_KV_TOKENIZER_PAD_ID, &vocab.special_pad_id }, + const std::vector> special_token_types = { + { LLM_KV_TOKENIZER_BOS_ID, vocab.special_bos_id }, + { LLM_KV_TOKENIZER_EOS_ID, vocab.special_eos_id }, + { LLM_KV_TOKENIZER_UNK_ID, vocab.special_unk_id }, + { LLM_KV_TOKENIZER_SEP_ID, vocab.special_sep_id }, + { LLM_KV_TOKENIZER_PAD_ID, vocab.special_pad_id }, }; - for (auto & it : special_token_types ) { - int32_t id = -1; - const std::string kstr = kv(std::get<0>(it)); + for (const auto & it : special_token_types ) { + const std::string key = kv(std::get<0>(it)); + int32_t & id = std::get<1>(it), old_id = id; - GGUF_GET_KEY(ctx, id, gguf_get_val_u32, GGUF_TYPE_UINT32, false, kstr); + GGUF_GET_KEY(ctx, id, gguf_get_val_u32, GGUF_TYPE_UINT32, false, key); if (id != -1 && (id < 0 || size_t(id) >= vocab.id_to_token.size())) { - LLAMA_LOG_WARN("%s: bad special token value %d for key '%s' -- ignoring\n", __func__, id, kstr.c_str()); - continue; + LLAMA_LOG_WARN("%s: bad special token: '%s' = %d, using default id %d\n", + __func__, key.c_str(), id, old_id); + id = old_id; } - *(std::get<1>(it)) = id; } } From 22b914e0ba6071791739c1c478c5df6b37f50633 Mon Sep 17 00:00:00 2001 From: KerfuffleV2 Date: Tue, 17 Oct 2023 03:54:37 -0600 Subject: [PATCH 5/8] Minor code cleanups. --- llama.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/llama.cpp b/llama.cpp index 8892691613f..5e18def98a6 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2242,19 +2242,22 @@ static void llm_load_vocab( // special tokens { - const std::vector> special_token_types = { + const std::vector> special_token_types = { { LLM_KV_TOKENIZER_BOS_ID, vocab.special_bos_id }, { LLM_KV_TOKENIZER_EOS_ID, vocab.special_eos_id }, { LLM_KV_TOKENIZER_UNK_ID, vocab.special_unk_id }, { LLM_KV_TOKENIZER_SEP_ID, vocab.special_sep_id }, { LLM_KV_TOKENIZER_PAD_ID, vocab.special_pad_id }, }; - for (const auto & it : special_token_types ) { + for (const auto & it : special_token_types) { const std::string key = kv(std::get<0>(it)); int32_t & id = std::get<1>(it), old_id = id; GGUF_GET_KEY(ctx, id, gguf_get_val_u32, GGUF_TYPE_UINT32, false, key); - if (id != -1 && (id < 0 || size_t(id) >= vocab.id_to_token.size())) { + // Must be >= -1 and < vocab size. Since the key is unsigned, -1 + // can only come from the default value, so there's no point in + // validating that. + if (size_t(id + 1) > vocab.id_to_token.size()) { LLAMA_LOG_WARN("%s: bad special token: '%s' = %d, using default id %d\n", __func__, key.c_str(), id, old_id); id = old_id; @@ -6101,7 +6104,7 @@ static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) { } static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) { - const char * hex = "0123456789ABCDEF"; + static const char * hex = "0123456789ABCDEF"; switch (llama_vocab_get_type(vocab)) { case LLAMA_VOCAB_TYPE_SPM: { const char buf[7] = { '<', '0', 'x', hex[ch >> 4], hex[ch & 15], '>', 0 }; From 3a007e2c81193ce233a6ef728449b50a6e86fa90 Mon Sep 17 00:00:00 2001 From: KerfuffleV2 Date: Tue, 17 Oct 2023 04:30:13 -0600 Subject: [PATCH 6/8] Fix convert.py error msg when added tokens are out of range --- convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert.py b/convert.py index e9b08d344f5..f506546c058 100755 --- a/convert.py +++ b/convert.py @@ -369,7 +369,7 @@ def __init__(self, fname_tokenizer: Path, fname_added_tokens: Path | None) -> No expected_ids = list(range(vocab_size, vocab_size + len(added_tokens))) actual_ids = sorted(added_tokens.values()) if expected_ids != actual_ids: - raise Exception(f"Expected added token IDs to be sequential and start at {len(added_tokens)}; got {actual_ids}") + raise Exception(f"Expected added token IDs to be sequential and start at {vocab_size}; got {actual_ids}") items = sorted(added_tokens.items(), key=lambda text_idx: text_idx[1]) self.added_tokens_list = [text for (text, idx) in items] From 8796025b46efd76ae10dc5c12affeae5d8aaf4f7 Mon Sep 17 00:00:00 2001 From: KerfuffleV2 Date: Tue, 17 Oct 2023 05:02:41 -0600 Subject: [PATCH 7/8] Make gguf SpecialVocab vocab size-aware Update conversion scripts accordingly --- convert-baichuan-hf-to-gguf.py | 2 +- convert-bloom-hf-to-gguf.py | 2 +- convert-falcon-hf-to-gguf.py | 2 +- convert-gptneox-hf-to-gguf.py | 2 +- convert-llama-ggml-to-gguf.py | 4 +++- convert-mpt-hf-to-gguf.py | 2 +- convert-refact-hf-to-gguf.py | 2 +- convert-starcoder-hf-to-gguf.py | 2 +- convert.py | 11 +++++++--- gguf-py/gguf/gguf.py | 36 +++++++++++++++++++++++---------- 10 files changed, 43 insertions(+), 22 deletions(-) diff --git a/convert-baichuan-hf-to-gguf.py b/convert-baichuan-hf-to-gguf.py index 513a7516a25..9508904fae2 100755 --- a/convert-baichuan-hf-to-gguf.py +++ b/convert-baichuan-hf-to-gguf.py @@ -224,7 +224,7 @@ def parse_args() -> argparse.Namespace: gguf_writer.add_token_scores(scores) gguf_writer.add_token_types(toktypes) -special_vocab = gguf.SpecialVocab(dir_model) +special_vocab = gguf.SpecialVocab(dir_model, n_vocab = len(tokens)) special_vocab.add_to_gguf(gguf_writer) # TENSORS diff --git a/convert-bloom-hf-to-gguf.py b/convert-bloom-hf-to-gguf.py index 7bfc95ec11d..14dbd793c84 100755 --- a/convert-bloom-hf-to-gguf.py +++ b/convert-bloom-hf-to-gguf.py @@ -129,7 +129,7 @@ def parse_args() -> argparse.Namespace: gguf_writer.add_token_scores(scores) gguf_writer.add_token_types(toktypes) -special_vocab = gguf.SpecialVocab(dir_model, load_merges=True) +special_vocab = gguf.SpecialVocab(dir_model, load_merges=True, n_vocab = len(tokens)) special_vocab.add_to_gguf(gguf_writer) # TENSORS diff --git a/convert-falcon-hf-to-gguf.py b/convert-falcon-hf-to-gguf.py index 9252e1c46a7..d4d1fac7537 100755 --- a/convert-falcon-hf-to-gguf.py +++ b/convert-falcon-hf-to-gguf.py @@ -145,7 +145,7 @@ def parse_args() -> argparse.Namespace: gguf_writer.add_token_scores(scores) gguf_writer.add_token_types(toktypes) -special_vocab = gguf.SpecialVocab(dir_model, load_merges = True) +special_vocab = gguf.SpecialVocab(dir_model, load_merges = True, n_vocab = len(tokens)) special_vocab.add_to_gguf(gguf_writer) # TENSORS diff --git a/convert-gptneox-hf-to-gguf.py b/convert-gptneox-hf-to-gguf.py index d4e85f51845..f1599b0c44e 100755 --- a/convert-gptneox-hf-to-gguf.py +++ b/convert-gptneox-hf-to-gguf.py @@ -134,7 +134,7 @@ def parse_args() -> argparse.Namespace: gguf_writer.add_token_scores(scores) gguf_writer.add_token_types(toktypes) -special_vocab = gguf.SpecialVocab(dir_model, load_merges = True) +special_vocab = gguf.SpecialVocab(dir_model, load_merges = True, n_vocab = len(tokens)) special_vocab.add_to_gguf(gguf_writer) # TENSORS diff --git a/convert-llama-ggml-to-gguf.py b/convert-llama-ggml-to-gguf.py index b5d3e0b3c3a..871add64d4c 100755 --- a/convert-llama-ggml-to-gguf.py +++ b/convert-llama-ggml-to-gguf.py @@ -388,7 +388,9 @@ def handle_metadata(cfg, hp): cfg.vocab_dir if cfg.vocab_dir is not None else cfg.model_metadata_dir, cfg.vocabtype ) # FIXME: Respect cfg.vocab_dir? - svocab = gguf.SpecialVocab(cfg.model_metadata_dir) + svocab = gguf.SpecialVocab(cfg.model_metadata_dir, + load_merges = cfg.vocabtype == 'bpe', + n_vocab = vocab.vocab_size) convert.check_vocab_size(params, vocab) return (params, vocab, svocab) diff --git a/convert-mpt-hf-to-gguf.py b/convert-mpt-hf-to-gguf.py index 19a66820dce..21b9fd5071b 100755 --- a/convert-mpt-hf-to-gguf.py +++ b/convert-mpt-hf-to-gguf.py @@ -139,7 +139,7 @@ def parse_args() -> argparse.Namespace: gguf_writer.add_token_scores(scores) gguf_writer.add_token_types(toktypes) -special_vocab = gguf.SpecialVocab(dir_model, load_merges = True) +special_vocab = gguf.SpecialVocab(dir_model, load_merges = True, n_vocab = len(tokens)) special_vocab.add_to_gguf(gguf_writer) # TENSORS diff --git a/convert-refact-hf-to-gguf.py b/convert-refact-hf-to-gguf.py index bfeabc0825b..934f3852b24 100755 --- a/convert-refact-hf-to-gguf.py +++ b/convert-refact-hf-to-gguf.py @@ -150,7 +150,7 @@ def parse_args() -> argparse.Namespace: gguf_writer.add_token_scores(scores) gguf_writer.add_token_types(toktypes) -special_vocab = gguf.SpecialVocab(dir_model, load_merges=True) +special_vocab = gguf.SpecialVocab(dir_model, load_merges=True, n_vocab = len(tokens)) special_vocab.add_to_gguf(gguf_writer) # TENSORS diff --git a/convert-starcoder-hf-to-gguf.py b/convert-starcoder-hf-to-gguf.py index 90fa0c32fbd..fe8815cbf6f 100755 --- a/convert-starcoder-hf-to-gguf.py +++ b/convert-starcoder-hf-to-gguf.py @@ -122,7 +122,7 @@ def parse_args() -> argparse.Namespace: gguf_writer.add_token_scores(scores) gguf_writer.add_token_types(toktypes) -special_vocab = gguf.SpecialVocab(dir_model, load_merges = True) +special_vocab = gguf.SpecialVocab(dir_model, load_merges = True, n_vocab = len(tokens)) special_vocab.add_to_gguf(gguf_writer) # TENSORS diff --git a/convert.py b/convert.py index f506546c058..b155ab8694d 100755 --- a/convert.py +++ b/convert.py @@ -1159,10 +1159,13 @@ def main(args_in: list[str] | None = None) -> None: vocab: Vocab if args.vocab_only: - assert args.outfile, "need --outfile if using --vocab-only" + if not args.outfile: + raise ValueError("need --outfile if using --vocab-only") # FIXME: Try to respect vocab_dir somehow? vocab = load_vocab(args.vocab_dir or args.model, args.vocabtype) - special_vocab = gguf.SpecialVocab(model_plus.paths[0].parent, load_merges = args.vocabtype == 'bpe') + special_vocab = gguf.SpecialVocab(model_plus.paths[0].parent, + load_merges = args.vocabtype == 'bpe', + n_vocab = vocab.vocab_size) outfile = args.outfile OutputFile.write_vocab_only(outfile, params, vocab, special_vocab) print(f"Wrote {outfile}") @@ -1174,7 +1177,9 @@ def main(args_in: list[str] | None = None) -> None: vocab_dir = args.vocab_dir if args.vocab_dir else model_plus.paths[0].parent vocab = load_vocab(vocab_dir, args.vocabtype) # FIXME: Try to respect vocab_dir somehow? - special_vocab = gguf.SpecialVocab(model_plus.paths[0].parent, load_merges = args.vocabtype == 'bpe') + special_vocab = gguf.SpecialVocab(model_plus.paths[0].parent, + load_merges = args.vocabtype == 'bpe', + n_vocab = vocab.vocab_size) model = model_plus.model model = convert_model_names(model, params) diff --git a/gguf-py/gguf/gguf.py b/gguf-py/gguf/gguf.py index 557ce7ac017..ecdc7a3b679 100644 --- a/gguf-py/gguf/gguf.py +++ b/gguf-py/gguf/gguf.py @@ -968,12 +968,15 @@ class SpecialVocab: merges: list[str] = [] special_token_types: tuple[str, ...] = ('bos', 'eos', 'unk', 'sep', 'pad') special_token_ids: dict[str, int] = {} + n_vocab: int | None = None def __init__( self, path: str | os.PathLike[str], load_merges: bool = False, special_token_types: tuple[str, ...] | None = None, + n_vocab: int | None = None, ): self.special_token_ids = {} + self.n_vocab = n_vocab self.load_merges = load_merges if special_token_types is not None: self.special_token_types = special_token_types @@ -983,6 +986,16 @@ def _load(self, path: Path) -> None: if not self._try_load_from_tokenizer_json(path): self._try_load_from_config_json(path) + def _set_special_token(self, typ: str, tid: Any): + if not isinstance(tid, int) or tid < 0: + return + if self.n_vocab is None or tid < self.n_vocab: + self.special_token_ids[typ] = tid + return + print(f'gguf: WARNING: Special token type {typ}, id {tid} out of range, must be under {self.n_vocab} - skipping', + file = sys.stderr) + + def _try_load_from_tokenizer_json(self, path: Path) -> bool: tokenizer_file = path / 'tokenizer.json' if not tokenizer_file.is_file(): @@ -1010,10 +1023,11 @@ def _try_load_from_tokenizer_json(self, path: Path) -> bool: tc_content = entry_content else: continue - for maybe_token_id in (atok.get('id') for atok in added_tokens if atok.get('content') == tc_content): - if isinstance(maybe_token_id, int) and maybe_token_id >= 0: - self.special_token_ids[typ] = maybe_token_id - break + # We only need the first match here. + maybe_token_id = next(( + atok.get('id') for atok in added_tokens + if atok.get('content') == tc_content), None) + self._set_special_token(typ, maybe_token_id) return True def _try_load_from_config_json(self, path: Path) -> bool: @@ -1023,21 +1037,21 @@ def _try_load_from_config_json(self, path: Path) -> bool: with open(config_file, encoding = 'utf-8') as f: config = json.load(f) for typ in self.special_token_types: - maybe_token_id = config.get(f'{typ}_token_id') - if isinstance(maybe_token_id, int) and maybe_token_id >= 0: - self.special_token_ids[typ] = maybe_token_id + self._set_special_token(typ, config.get(f'{typ}_token_id')) return True - def add_to_gguf(self, gw: GGUFWriter) -> None: + def add_to_gguf(self, gw: GGUFWriter, quiet: bool = False) -> None: if len(self.merges) > 0: - print(f'gguf: Adding {len(self.merges)} merge(s).') + if not quiet: + print(f'gguf: Adding {len(self.merges)} merge(s).') gw.add_token_merges(self.merges) for typ, tokid in self.special_token_ids.items(): handler: Callable[[int], None] | None = getattr(gw, f'add_{typ}_token_id', None) if handler is None: - print(f'gguf: WARNING: No handler for special token type {typ} with id {tokid} - skipping') + print(f'gguf: WARNING: No handler for special token type {typ} with id {tokid} - skipping', file = sys.stderr) continue - print(f'gguf: Setting special token type {typ} to {tokid}') + if not quiet: + print(f'gguf: Setting special token type {typ} to {tokid}') handler(tokid) def __repr__(self) -> str: From 76b05fc4a0052ac0b7202e002893d6cdd526daab Mon Sep 17 00:00:00 2001 From: Kerfuffle <44031344+KerfuffleV2@users.noreply.github.com> Date: Tue, 17 Oct 2023 12:13:46 -0600 Subject: [PATCH 8/8] Avoid a string copy Co-authored-by: Georgi Gerganov --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 5e18def98a6..bbe01a70b92 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2250,7 +2250,7 @@ static void llm_load_vocab( { LLM_KV_TOKENIZER_PAD_ID, vocab.special_pad_id }, }; for (const auto & it : special_token_types) { - const std::string key = kv(std::get<0>(it)); + const std::string & key = kv(std::get<0>(it)); int32_t & id = std::get<1>(it), old_id = id; GGUF_GET_KEY(ctx, id, gguf_get_val_u32, GGUF_TYPE_UINT32, false, key);