# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import cache
import torch
from transformers import AutoTokenizer
class CosyVoice2Tokenizer:
def __init__(self, token_path, skip_special_tokens=True):
super().__init__()
# NOTE: non-chat model, all these special tokens keep randomly initialized.
special_tokens = {
"eos_token": "<|endoftext|>",
"pad_token": "<|endoftext|>",
"additional_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|endofprompt|>",
"[breath]",
"",
"",
"[noise]",
"[laughter]",
"[cough]",
"[clucking]",
"[accent]",
"[quick_breath]",
"",
"",
"[hissing]",
"[sigh]",
"[vocalized-noise]",
"[lipsmack]",
"[mn]",
],
}
self.special_tokens = special_tokens
self.tokenizer = AutoTokenizer.from_pretrained(token_path)
self.tokenizer.add_special_tokens(special_tokens)
self.skip_special_tokens = skip_special_tokens
def encode(self, text, **kwargs):
tokens = self.tokenizer([text], return_tensors="pt")
tokens = tokens["input_ids"][0].cpu().tolist()
return tokens
def decode(self, tokens):
tokens = torch.tensor(tokens, dtype=torch.int64)
text = self.tokenizer.batch_decode([tokens], skip_special_tokens=self.skip_special_tokens)[0]
return text
class CosyVoice3Tokenizer(CosyVoice2Tokenizer):
def __init__(self, token_path, skip_special_tokens=True):
# NOTE: non-chat model, all these special tokens keep randomly initialized.
special_tokens = {
"eos_token": "<|endoftext|>",
"pad_token": "<|endoftext|>",
"additional_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|endofprompt|>",
"[breath]",
"",
"",
"[noise]",
"[laughter]",
"[cough]",
"[clucking]",
"[accent]",
"[quick_breath]",
"",
"",
"[hissing]",
"[sigh]",
"[vocalized-noise]",
"[lipsmack]",
"[mn]",
"<|endofsystem|>",
"[AA]",
"[AA0]",
"[AA1]",
"[AA2]",
"[AE]",
"[AE0]",
"[AE1]",
"[AE2]",
"[AH]",
"[AH0]",
"[AH1]",
"[AH2]",
"[AO]",
"[AO0]",
"[AO1]",
"[AO2]",
"[AW]",
"[AW0]",
"[AW1]",
"[AW2]",
"[AY]",
"[AY0]",
"[AY1]",
"[AY2]",
"[B]",
"[CH]",
"[D]",
"[DH]",
"[EH]",
"[EH0]",
"[EH1]",
"[EH2]",
"[ER]",
"[ER0]",
"[ER1]",
"[ER2]",
"[EY]",
"[EY0]",
"[EY1]",
"[EY2]",
"[F]",
"[G]",
"[HH]",
"[IH]",
"[IH0]",
"[IH1]",
"[IH2]",
"[IT]",
"[IT0]",
"[IT1]",
"[IT2]",
"[JH]",
"[K]",
"[L]",
"[M]",
"[N]",
"[NG]",
"[OW]",
"[OW0]",
"[OW1]",
"[OW2]",
"[OY]",
"[OY0]",
"[OY1]",
"[OY2]",
"[P]",
"[R]",
"[S]",
"[SH]",
"[T]",
"[TH]",
"[UH]",
"[UH0]",
"[UH1]",
"[UH2]",
"[UW]",
"[UW0]",
"[UW1]",
"[UW2]",
"[V]",
"[W]",
"[Y]",
"[Z]",
"[ZH]",
"[a]",
"[ai]",
"[an]",
"[ang]",
"[ao]",
"[b]",
"[c]",
"[ch]",
"[d]",
"[e]",
"[ei]",
"[en]",
"[eng]",
"[f]",
"[g]",
"[h]",
"[i]",
"[ian]",
"[in]",
"[ing]",
"[iu]",
"[ià]",
"[iàn]",
"[iàng]",
"[iào]",
"[iá]",
"[ián]",
"[iáng]",
"[iáo]",
"[iè]",
"[ié]",
"[iòng]",
"[ióng]",
"[iù]",
"[iú]",
"[iā]",
"[iān]",
"[iāng]",
"[iāo]",
"[iē]",
"[iě]",
"[iōng]",
"[iū]",
"[iǎ]",
"[iǎn]",
"[iǎng]",
"[iǎo]",
"[iǒng]",
"[iǔ]",
"[j]",
"[k]",
"[l]",
"[m]",
"[n]",
"[o]",
"[ong]",
"[ou]",
"[p]",
"[q]",
"[r]",
"[s]",
"[sh]",
"[t]",
"[u]",
"[uang]",
"[ue]",
"[un]",
"[uo]",
"[uà]",
"[uài]",
"[uàn]",
"[uàng]",
"[uá]",
"[uái]",
"[uán]",
"[uáng]",
"[uè]",
"[ué]",
"[uì]",
"[uí]",
"[uò]",
"[uó]",
"[uā]",
"[uāi]",
"[uān]",
"[uāng]",
"[uē]",
"[uě]",
"[uī]",
"[uō]",
"[uǎ]",
"[uǎi]",
"[uǎn]",
"[uǎng]",
"[uǐ]",
"[uǒ]",
"[vè]",
"[w]",
"[x]",
"[y]",
"[z]",
"[zh]",
"[à]",
"[ài]",
"[àn]",
"[àng]",
"[ào]",
"[á]",
"[ái]",
"[án]",
"[áng]",
"[áo]",
"[è]",
"[èi]",
"[èn]",
"[èng]",
"[èr]",
"[é]",
"[éi]",
"[én]",
"[éng]",
"[ér]",
"[ì]",
"[ìn]",
"[ìng]",
"[í]",
"[ín]",
"[íng]",
"[ò]",
"[òng]",
"[òu]",
"[ó]",
"[óng]",
"[óu]",
"[ù]",
"[ùn]",
"[ú]",
"[ún]",
"[ā]",
"[āi]",
"[ān]",
"[āng]",
"[āo]",
"[ē]",
"[ēi]",
"[ēn]",
"[ēng]",
"[ě]",
"[ěi]",
"[ěn]",
"[ěng]",
"[ěr]",
"[ī]",
"[īn]",
"[īng]",
"[ō]",
"[ōng]",
"[ōu]",
"[ū]",
"[ūn]",
"[ǎ]",
"[ǎi]",
"[ǎn]",
"[ǎng]",
"[ǎo]",
"[ǐ]",
"[ǐn]",
"[ǐng]",
"[ǒ]",
"[ǒng]",
"[ǒu]",
"[ǔ]",
"[ǔn]",
"[ǘ]",
"[ǚ]",
"[ǜ]",
],
}
self.special_tokens = special_tokens
self.tokenizer = AutoTokenizer.from_pretrained(token_path)
self.tokenizer.add_special_tokens(special_tokens)
self.skip_special_tokens = skip_special_tokens
@cache
def get_qwen_tokenizer(token_path: str, skip_special_tokens: bool, version: str = "cosyvoice3"):
if version == "cosyvoice2":
return CosyVoice2Tokenizer(token_path=token_path, skip_special_tokens=skip_special_tokens)
elif version == "cosyvoice3":
return CosyVoice3Tokenizer(token_path=token_path, skip_special_tokens=skip_special_tokens)
else:
raise ValueError