diff --git a/tokenization_moonshot.py b/tokenization_moonshot.py index a0950f0..2dfe8f5 100644 --- a/tokenization_moonshot.py +++ b/tokenization_moonshot.py @@ -17,7 +17,7 @@ from tiktoken.load import load_tiktoken_bpe from tokenizers import AddedToken from transformers.tokenization_utils import PreTrainedTokenizer from transformers.utils import to_py_obj -from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode +from transformers.convert_slow_tokenizer import bytes_to_unicode logger = getLogger(__name__)