From 9ec7bbe9c7872a58b925639805d7e95e007de298 Mon Sep 17 00:00:00 2001 From: Karl-Johan Alm Date: Fri, 8 Aug 2025 14:50:22 +0900 Subject: [PATCH 1/5] test adapters --- tests/gated-tokenizers | 1 + tests/test_autoguess.py | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) create mode 120000 tests/gated-tokenizers diff --git a/tests/gated-tokenizers b/tests/gated-tokenizers new file mode 120000 index 00000000000..bc4123948cd --- /dev/null +++ b/tests/gated-tokenizers @@ -0,0 +1 @@ +../../gated-tokenizers \ No newline at end of file diff --git a/tests/test_autoguess.py b/tests/test_autoguess.py index 1ea9ed70ba0..0020fc13583 100644 --- a/tests/test_autoguess.py +++ b/tests/test_autoguess.py @@ -113,9 +113,9 @@ def templ(rolelist): expect = system("SyS-tEm") templated = templ([{"role": "system", "content": "SyS-tEm"}, {"role": "user", "content": "user"}]) if expect not in templated: - return False, f"system role missing expected fragment {expect.replace("\n", "\\n")}: {templated.replace("\n", "\\n")}" + return False, f"system role missing expected fragment\n\tadapter: {expect.replace("\n", "\\n")}\n\ttokenizer: {templated.replace("\n", "\\n")}" - # Test user/asst/usernvidia/Llama-4-Scout-17B-16E-Instruct-FP8 + # Test user/asst/user expect = [ user("user_1"), assistant("asst_1"), @@ -129,17 +129,21 @@ def templ(rolelist): rem = templated for sub in expect: if sub not in rem: - return False, f"missing expected fragment {sub.replace("\n", "\\n")}: {rem.replace("\n", "\\n")}" + return False, f"missing expected fragment\n\tadapter: {sub.replace("\n", "\\n")}\n\ttokenizer: {rem.replace("\n", "\\n")}" rem = rem.split(sub, 1)[1] except jinja2.exceptions.TemplateError as e: return False, f"template error: {e}" return True, None +filter = sys.argv[1] if len(sys.argv) > 1 else None + failures = 0 seen = set() namefmt = "{name:<" + str(max(len(name) for name in AUTOGUESS_MAPPING.keys())) + "}" hmifmt = "{huggingface_model_id:<" + str(max(len(huggingface_model_id) for huggingface_model_id in AUTOGUESS_MAPPING.values())) + "}" for name, huggingface_model_id in AUTOGUESS_MAPPING.items(): + if filter and filter not in name: + continue seen.add(name) if huggingface_model_id == "***UNKNOWN***": print(namefmt.format(name=name) + " = " + namefmt.format(name="***UNKNOWN***") + " : PENDING") @@ -162,10 +166,11 @@ def templ(rolelist): print(namefmt.format(name=name) + " = " + namefmt.format(name=matched) + " : " + ("OK " if adaptercheck and name == matched else reason if not adaptercheck else "FAILURE") + " " + hmifmt.format(huggingface_model_id=huggingface_model_id) + " " + sub_template) failures += name != matched or not adaptercheck -for entry in autoguess: - if entry['name'] not in seen: - print(namefmt.format(name=entry['name']) + " MISSING MAPPING") - failures += 1 +if filter is None: + for entry in autoguess: + if entry['name'] not in seen: + print(namefmt.format(name=entry['name']) + " MISSING MAPPING") + failures += 1 if failures > 0: print(f"There were {failures} failure(s)!") From 04f4332c32dd2a1b37f42c4fcb904712704dc497 Mon Sep 17 00:00:00 2001 From: Karl-Johan Alm Date: Fri, 8 Aug 2025 14:58:30 +0900 Subject: [PATCH 2/5] add assistant_gen adapter key --- kcpp_adapters/AutoGuess.json | 11 +++++++---- koboldcpp.py | 9 ++++++--- tests/test_autoguess.py | 6 +++--- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/kcpp_adapters/AutoGuess.json b/kcpp_adapters/AutoGuess.json index ee7f598668f..d9e57f8efc0 100644 --- a/kcpp_adapters/AutoGuess.json +++ b/kcpp_adapters/AutoGuess.json @@ -106,7 +106,8 @@ "system_end": "[/INST]", "user_start": "[INST] ", "user_end": "", - "assistant_start": "[/INST]", + "assistant_start": "[/INST] ", + "assistant_gen": "[/INST]", "assistant_end": "" } }, { @@ -168,11 +169,12 @@ "search": ["<|bom|>","is_last_checked_defined"], "name": "Jamba", "adapter": { - "system_start": "<|bom|><|system|>", + "system_start": "<|bom|><|system|> ", "system_end": "<|eom|>", - "user_start": "<|bom|><|user|>", + "user_start": "<|bom|><|user|> ", "user_end": "<|eom|>", - "assistant_start": "<|bom|><|assistant|>", + "assistant_start": "<|bom|><|assistant|> ", + "assistant_gen": "<|bom|><|assistant|>", "assistant_end": "<|eom|>" } }, { @@ -206,6 +208,7 @@ "user_start": "User: ", "user_end": "\n\n", "assistant_start": "Assistant: ", + "assistant_gen": "Assistant:", "assistant_end": "\n\n" } }, { diff --git a/koboldcpp.py b/koboldcpp.py index 85f74333b04..10fe101ff5c 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -2419,6 +2419,7 @@ def transform_genparams(genparams, api_format): user_message_end = adapter_obj.get("user_end", "") assistant_message_start = adapter_obj.get("assistant_start", "\n### Response:\n") assistant_message_end = adapter_obj.get("assistant_end", "") + assistant_message_gen = adapter_obj.get("assistant_gen", assistant_message_start) tools_message_start = adapter_obj.get("tools_start", "\nTool Results:\n") tools_message_end = adapter_obj.get("tools_end", "") images_added = [] @@ -2531,7 +2532,7 @@ def transform_genparams(genparams, api_format): elif message['role'] == "tool": messages_string += tools_message_end - messages_string += assistant_message_start + messages_string += assistant_message_gen genparams["prompt"] = messages_string if len(images_added)>0: genparams["images"] = images_added @@ -2552,7 +2553,8 @@ def transform_genparams(genparams, api_format): adapter_obj = {} if chatcompl_adapter is None else chatcompl_adapter user_message_start = adapter_obj.get("user_start", "### Instruction:") assistant_message_start = adapter_obj.get("assistant_start", "### Response:") - genparams["prompt"] = f"{user_message_start} In one sentence, write a descriptive caption for this image.\n{assistant_message_start}" + assistant_message_gen = adapter_obj.get("assistant_gen", assistant_message_start) + genparams["prompt"] = f"{user_message_start} In one sentence, write a descriptive caption for this image.\n{assistant_message_gen}" elif api_format==6: detokstr = "" @@ -2560,12 +2562,13 @@ def transform_genparams(genparams, api_format): adapter_obj = {} if chatcompl_adapter is None else chatcompl_adapter user_message_start = adapter_obj.get("user_start", "\n\n### Instruction:\n") assistant_message_start = adapter_obj.get("assistant_start", "\n\n### Response:\n") + assistant_message_gen = adapter_obj.get("assistant_gen", assistant_message_start) try: detokstr = detokenize_ids(tokids) except Exception as e: utfprint("Ollama Context Error: " + str(e)) ollamasysprompt = genparams.get('system', "") - ollamabodyprompt = f"{detokstr}{user_message_start}{genparams.get('prompt', '')}{assistant_message_start}" + ollamabodyprompt = f"{detokstr}{user_message_start}{genparams.get('prompt', '')}{assistant_message_gen}" ollamaopts = genparams.get('options', {}) if genparams.get('stop',[]) is not None: genparams["stop_sequence"] = genparams.get('stop', []) diff --git a/tests/test_autoguess.py b/tests/test_autoguess.py index 0020fc13583..0b68ed6061d 100644 --- a/tests/test_autoguess.py +++ b/tests/test_autoguess.py @@ -21,8 +21,8 @@ "Google Gemma 3n": "lmstudio-community/gemma-3n-E4B-it-MLX-bf16", "Llama 3.x": "Steelskull/L3.3-Shakudo-70b", "Llama 4": "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8", - "Mistral V7 (with system prompt)": "Doctor-Shotgun/MS3.2-24B-Magnum-Diamond", - "Mistral V3": "mistralai/Mistral-7B-Instruct-v0.3", + "Mistral Tekken": "Doctor-Shotgun/MS3.2-24B-Magnum-Diamond", + "Mistral Non-Tekken": "mistralai/Mistral-7B-Instruct-v0.3", "GLM-4": "THUDM/glm-4-9b-chat-hf", "Phi 3.5": "microsoft/Phi-3.5-mini-instruct", "Phi 4 (mini)": "microsoft/Phi-4-mini-instruct", @@ -36,7 +36,7 @@ } AUTOGUESS_SKIP_ADAPTER_TESTS = { - "Mistral V3": {"system"}, # Poor system support + "Mistral Non-Tekken": {"system"}, # Poor system support "Mistral (Generic)": {"system"}, # Poor system support } From 09dac6ab7e95e301f1b6f9af168351ba5894471a Mon Sep 17 00:00:00 2001 From: Karl-Johan Alm Date: Fri, 8 Aug 2025 15:03:54 +0900 Subject: [PATCH 3/5] add support for chat templates stored as .jinja files --- tests/test_autoguess.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_autoguess.py b/tests/test_autoguess.py index 0b68ed6061d..df4b8b80ba3 100644 --- a/tests/test_autoguess.py +++ b/tests/test_autoguess.py @@ -31,6 +31,7 @@ "Jamba": "ai21labs/Jamba-tiny-dev", "Dots": "rednote-hilab/dots.llm1.inst", "RWKV World": "fla-hub/rwkv7-1.5B-world", + "OpenAI Harmony": "openai/gpt-oss-120b", "Mistral (Generic)": "mistralai/Mistral-Nemo-Instruct-2407", "ChatML (Generic)": "NewEden/Gemma-27B-chatml", } @@ -58,10 +59,12 @@ def get_tokenizer_config_for_huggingface_model_id(huggingface_model_id: str): with open(fname) as f: return json.load(f) - for filename in ["tokenizer_config.json", "chat_template.json"]: + for filename in ["tokenizer_config.json", "chat_template.json", "chat_template.jinja"]: url = f"https://huggingface.co/{huggingface_model_id}/resolve/main/{filename}" response = requests.get(url) if response.status_code == 200: + if url.endswith(".jinja"): + return {"chat_template": response.text} v = json.loads(response.text) if 'chat_template' in v: return v From 7efbea86ddb5a569f8a63612e68bad861f4fc6d9 Mon Sep 17 00:00:00 2001 From: Karl-Johan Alm Date: Sat, 9 Aug 2025 09:28:34 +0900 Subject: [PATCH 4/5] removed mistakenly commited gated-tokenizers link --- tests/gated-tokenizers | 1 - 1 file changed, 1 deletion(-) delete mode 120000 tests/gated-tokenizers diff --git a/tests/gated-tokenizers b/tests/gated-tokenizers deleted file mode 120000 index bc4123948cd..00000000000 --- a/tests/gated-tokenizers +++ /dev/null @@ -1 +0,0 @@ -../../gated-tokenizers \ No newline at end of file From 60aa8a2ac50834ee712f841770d930eb2763b502 Mon Sep 17 00:00:00 2001 From: Karl-Johan Alm Date: Sat, 9 Aug 2025 19:29:13 +0900 Subject: [PATCH 5/5] autoguess: Harmony: add missing newline prefixes to system_end --- kcpp_adapters/AutoGuess.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kcpp_adapters/AutoGuess.json b/kcpp_adapters/AutoGuess.json index d9e57f8efc0..b09b736f46e 100644 --- a/kcpp_adapters/AutoGuess.json +++ b/kcpp_adapters/AutoGuess.json @@ -193,7 +193,7 @@ "name": "OpenAI Harmony", "adapter": { "system_start": "<|start|>developer<|message|># Instructions\n\n", - "system_end": "<|end|>", + "system_end": "\n\n<|end|>", "user_start": "<|start|>user<|message|>", "user_end": "<|end|>", "assistant_start": "<|start|>assistant<|channel|>final<|message|>",