https://huggingface.co/docs/tokenizers/api/added-tokens

python
展开代码
from rich import print
from transformers import AutoTokenizer
def add_new_tokens(load_path: str, save_path: str, new_tokens: list[str]):
    tokenizer = AutoTokenizer.from_pretrained(load_path)
    tokenizer.add_tokens(new_tokens)
    tokenizer.save_pretrained(save_path)
def update_tokenizer(model_path: str):
    add_new_tokens(
        model_path, model_path, ["<|call_start|>", "<|call_end|>", "<|toolcall_start|>", "<|toolcall_end|>"]
    )
def test_tokenizer(model_path: str):
    tokenizer = AutoTokenizer.from_pretrained(model_path)
    text = "hello <|call_start|> world <|call_end|>"
    print(text, tokenizer.tokenize(text), tokenizer.encode(text))
    text = "hello <|toolcall_start|> world <|toolcall_end|>"
    print(text, tokenizer.tokenize(text), tokenizer.encode(text))
if __name__ == "__main__":
    update_tokenizer("/mnt/jfs6/model_ok/qwen2vl-0811-1/checkpoint-6400")
    test_tokenizer("/mnt/jfs6/model_ok/qwen2vl-0811-1/checkpoint-6400")

added_tokens.json 改变了：

bash
展开代码
root@hello-9tgxb-1867836-worker-0:/mnt/jfs6/model_ok/qwen2vl-0811-1/checkpoint-6400# cat added_tokens.json
{
  "<|box_end|>": 151649,
  "<|box_start|>": 151648,
  "<|call_end|>": 151660,
  "<|call_start|>": 151659,
  "<|endoftext|>": 151643,
  "<|im_end|>": 151645,
  "<|im_start|>": 151644,
  "<|image_pad|>": 151655,
  "<|object_ref_end|>": 151647,
  "<|object_ref_start|>": 151646,
  "<|quad_end|>": 151651,
  "<|quad_start|>": 151650,
  "<|toolcall_end|>": 151658,
  "<|toolcall_start|>": 151657,
  "<|video_pad|>": 151656,
  "<|vision_end|>": 151653,
  "<|vision_pad|>": 151654,
  "<|vision_start|>": 151652
}

tokenizer.json 和 tokenizer_config.json 也相应更新