mirror of
https://github.com/modelscope/FunASR
synced 2025-09-15 14:48:36 +08:00
17 lines
371 B
Python
17 lines
371 B
Python
|
|
try:
|
|
from transformers import AutoTokenizer
|
|
except:
|
|
# print("If you want to use hugging, please `pip install -U transformers`")
|
|
pass
|
|
|
|
from funasr.register import tables
|
|
|
|
@tables.register("tokenizer_classes", "HuggingfaceTokenizer")
|
|
def HuggingfaceTokenizer(init_param_path, **kwargs):
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained(init_param_path)
|
|
|
|
return tokenizer
|
|
|