speech_recognition/FunASR
1
0
Fork 0
mirror of https://github.com/modelscope/FunASR synced 2025-09-15 14:48:36 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #309 from alibaba-damo-academy/dev_lzr

Browse Source 
fix contextualparaformer bias_embed
This commit is contained in:
Xian Shi 2023-03-29 16:54:38 +08:00 committed by GitHub
commit 3852f61795
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
funasr/models/e2e_asr_paraformer.py
View File

@ -1085,6 +1085,7 @@ class ContextualParaformer(Paraformer):
inner_dim: int = 256,
bias_encoder_type: str = 'lstm',
label_bracket: bool = False,
use_decoder_embedding: bool = False,
):
assert check_argument_types()
assert 0.0 <= ctc_weight <= 1.0, ctc_weight
@ -1138,6 +1139,7 @@ class ContextualParaformer(Paraformer):
self.hotword_buffer = None
self.length_record = []
self.current_buffer_length = 0
self.use_decoder_embedding = use_decoder_embedding
def forward(
self,
@ -1279,7 +1281,10 @@ class ContextualParaformer(Paraformer):
hw_list.append(hw_tokens)
# padding
hw_list_pad = pad_list(hw_list, 0)
hw_embed = self.decoder.embed(hw_list_pad)
if self.use_decoder_embedding:
hw_embed = self.decoder.embed(hw_list_pad)
else:
hw_embed = self.bias_embed(hw_list_pad)
hw_embed, (_, _) = self.bias_encoder(hw_embed)
_ind = np.arange(0, len(hw_list)).tolist()
# update self.hotword_buffer, throw a part if oversize
@ -1395,13 +1400,19 @@ class ContextualParaformer(Paraformer):
# default hotword list
hw_list = [torch.Tensor([self.sos]).long().to(encoder_out.device)] # empty hotword list
hw_list_pad = pad_list(hw_list, 0)
hw_embed = self.bias_embed(hw_list_pad)
if self.use_decoder_embedding:
hw_embed = self.decoder.embed(hw_list_pad)
else:
hw_embed = self.bias_embed(hw_list_pad)
_, (h_n, _) = self.bias_encoder(hw_embed)
contextual_info = h_n.squeeze(0).repeat(encoder_out.shape[0], 1, 1)
else:
hw_lengths = [len(i) for i in hw_list]
hw_list_pad = pad_list([torch.Tensor(i).long() for i in hw_list], 0).to(encoder_out.device)
hw_embed = self.bias_embed(hw_list_pad)
if self.use_decoder_embedding:
hw_embed = self.decoder.embed(hw_list_pad)
else:
hw_embed = self.bias_embed(hw_list_pad)
hw_embed = torch.nn.utils.rnn.pack_padded_sequence(hw_embed, hw_lengths, batch_first=True,
enforce_sorted=False)
_, (h_n, _) = self.bias_encoder(hw_embed)