import logging from funasr.train.abs_model import AbsLM from funasr.train.abs_model import LanguageModel from funasr.models.seq_rnn_lm import SequentialRNNLM from funasr.models.transformer_lm import TransformerLM from funasr.torch_utils.initialize import initialize from funasr.train.class_choices import ClassChoices lm_choices = ClassChoices( "lm", classes=dict( seq_rnn=SequentialRNNLM, transformer=TransformerLM, ), type_check=AbsLM, default="seq_rnn", ) model_choices = ClassChoices( "model", classes=dict( lm=LanguageModel, ), default="lm", ) class_choices_list = [ # --lm and --lm_conf lm_choices, # --model and --model_conf model_choices ] def build_lm_model(args): # token_list if isinstance(args.token_list, str): with open(args.token_list, encoding="utf-8") as f: token_list = [line.rstrip() for line in f] args.token_list = list(token_list) vocab_size = len(token_list) logging.info(f"Vocabulary size: {vocab_size}") elif isinstance(args.token_list, (tuple, list)): token_list = list(args.token_list) vocab_size = len(token_list) logging.info(f"Vocabulary size: {vocab_size}") else: vocab_size = None # lm lm_class = lm_choices.get_class(args.lm) lm = lm_class(vocab_size=vocab_size, **args.lm_conf) args.model = args.model if hasattr(args, "model") else "lm" model_class = model_choices.get_class(args.model) model = model_class(lm=lm, vocab_size=vocab_size, **args.model_conf) # initialize if args.init is not None: initialize(model, args.init) return model