From 29431b31c89e79c10f8736e8f2742485ba1713d6 Mon Sep 17 00:00:00 2001 From: Joel Farthing Date: Tue, 30 Jun 2026 02:02:53 -0500 Subject: [PATCH] convert : recognize gpt-oss (o200k_harmony) tokenizer (#2060) The openai gpt-oss models, and the z-lab gpt-oss DFlash drafts that reuse their tokenizer, ship the o200k_harmony BPE tokenizer, which shares the GPT-4o pre-tokenizer regex already handled by LLAMA_VOCAB_PRE_TYPE_GPT4O. Its checksum was unrecognized, so get_vocab_base_pre() fell through to the "BPE pre-tokenizer was not recognized" warning and conversion failed. Register the gpt-oss-20b tokenizer hash against the existing "gpt-4o" pre-type in convert_hf_to_gguf_update.py and add the matching entry in get_vocab_base_pre(), following the dual-location pattern already used for the z-lab Qwen3.5 DFlash drafts. Co-authored-by: Joel Farthing <262452229+joelfarthing@users.noreply.github.com> --- convert_hf_to_gguf.py | 3 +++ convert_hf_to_gguf_update.py | 1 + 2 files changed, 4 insertions(+) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index a9c40c8c..fb683d36 100644 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -715,6 +715,9 @@ class Model: if chkhsh == "9dcf830ee9990cdbf78cc523a5f7bd9ad8f3f9890c2d3581d2785ad10f07049d": # ref: https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Base res = "mellum2" + if chkhsh == "ccc2ef013c104be7bae2965776d611e1d7a8a2a9c547dd93a682c9a9fc80352e": + # ref: https://huggingface.co/openai/gpt-oss-20b + res = "gpt-4o" if res is None: logger.warning("\n") logger.warning("**************************************************************************************") diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index 9168f903..0c9b6dfd 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -106,6 +106,7 @@ models = [ {"name": "grok-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/alvarobartt/grok-2-tokenizer", "chkhsh": "66b8d4e19ab16c3bfd89bce5d785fb7e0155e8648708a1f42077cb9fe002c273"}, {"name": "minimax-m2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/MiniMaxAI/MiniMax-M2", }, {"name": "mellum2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Base", }, + {"name": "gpt-4o", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openai/gpt-oss-20b", "chkhsh": "ccc2ef013c104be7bae2965776d611e1d7a8a2a9c547dd93a682c9a9fc80352e", }, # o200k_harmony shares the GPT-4o pre-tokenizer ]