diff --git a/tokenizer_config.json b/tokenizer_config.json index f1eb5e1..4ceb0a8 100644 --- a/tokenizer_config.json +++ b/tokenizer_config.json @@ -200,15 +200,8 @@ "<|end_of_image|>", "<|begin_of_video|>", "<|end_of_video|>", - "<|begin_of_box|>", - "<|end_of_box|>", - "<|sep|>", "<|image|>", - "<|video|>", - "", - "", - "", - "" + "<|video|>" ], "clean_up_tokenization_spaces": false, "do_lower_case": false,