speech_recognition/FunASR
1
0
Fork 0
mirror of https://github.com/modelscope/FunASR synced 2025-09-15 14:48:36 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

decoding

Browse Source
This commit is contained in:
游雁 2024-06-12 17:44:12 +08:00
parent 9afcf0ea7d
commit 2518f03d20
1 changed files with 22 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
funasr/models/llm_asr/model.py
View File

@ -427,9 +427,8 @@ class LLMASR2(nn.Module):
self.audio_encoder = audio_encoder self.audio_encoder = audio_encoder
# llm # llm
hub = llm_conf.get("hub", "hf")
self.llm = None self.llm = None
if hub == "hf":
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig
init_param_path = llm_conf.get("init_param_path", "vicuna-7b-v1.5") init_param_path = llm_conf.get("init_param_path", "vicuna-7b-v1.5")
@ -446,11 +445,18 @@ class LLMASR2(nn.Module):
param.requires_grad = False param.requires_grad = False
model.eval() model.eval()
self.llm = model self.llm = model
llm_dim = model.get_input_embeddings().weight.shape[-1]
# adaptor # adaptor
adaptor_class = tables.adaptor_classes.get(audio_adaptor) adaptor_class = tables.adaptor_classes.get(audio_adaptor)
audio_adaptor_conf["encoder_dim"] = audio_encoder_output_size audio_adaptor_conf["encoder_dim"] = audio_encoder_output_size
audio_adaptor_conf["llm_dim"] = llm_dim
audio_adaptor = adaptor_class(**audio_adaptor_conf) audio_adaptor = adaptor_class(**audio_adaptor_conf)
init_param_path = audio_adaptor_conf.get("init_param_path", None)
if init_param_path is not None:
src_state = torch.load(init_param_path, map_location="cpu")
flag = audio_adaptor.load_state_dict(src_state, strict=False)
logging.info(f"Loading audio_adaptor ckpt: {init_param_path}, status: {flag}")
self.audio_adaptor = audio_adaptor self.audio_adaptor = audio_adaptor