From f38d1057ba5fe9812c1936c3b6ba0b00cf3ed1f7 Mon Sep 17 00:00:00 2001 From: systemd Date: Fri, 30 Jan 2026 12:15:23 +0000 Subject: [PATCH] Transformers v5 support (#13) - Transformers v5 support (f2ab9d166e00f16237b0e09749df2faacc864d1e) Co-authored-by: Harry Mellor (batch 1/1) --- tokenization_moonshot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokenization_moonshot.py b/tokenization_moonshot.py index a0950f0..2dfe8f5 100644 --- a/tokenization_moonshot.py +++ b/tokenization_moonshot.py @@ -17,7 +17,7 @@ from tiktoken.load import load_tiktoken_bpe from tokenizers import AddedToken from transformers.tokenization_utils import PreTrainedTokenizer from transformers.utils import to_py_obj -from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode +from transformers.convert_slow_tokenizer import bytes_to_unicode logger = getLogger(__name__)