diff --git a/funasr/bin/asr_inference_paraformer_vad_punc.py b/funasr/bin/asr_inference_paraformer_vad_punc.py index 9001f1d6a..60780615d 100644 --- a/funasr/bin/asr_inference_paraformer_vad_punc.py +++ b/funasr/bin/asr_inference_paraformer_vad_punc.py @@ -529,8 +529,9 @@ def inference_modelscope( nbest=nbest, ) speech2text = Speech2Text(**speech2text_kwargs) - - text2punc = Text2Punc(punc_infer_config, punc_model_file, device=device, dtype=dtype) + text2punc = None + if punc_model_file is not None: + text2punc = Text2Punc(punc_infer_config, punc_model_file, device=device, dtype=dtype) if output_dir is not None: writer = DatadirWriter(output_dir) @@ -560,38 +561,28 @@ def inference_modelscope( allow_variable_data_keys=allow_variable_data_keys, inference=True, ) - - forward_time_total = 0.0 - length_total = 0.0 + finish_count = 0 file_count = 1 lfr_factor = 6 # 7 .Start for-loop asr_result_list = [] output_path = output_dir_v2 if output_dir_v2 is not None else output_dir + writer = None if output_path is not None: writer = DatadirWriter(output_path) ibest_writer = writer[f"1best_recog"] - # ibest_writer["punc_dict"][""] = " ".join(punc_infer_config.punc_list) - # ibest_writer["token_list"][""] = " ".join(asr_train_config.token_list) - else: - writer = None - + for keys, batch in loader: assert isinstance(batch, dict), type(batch) assert all(isinstance(s, str) for s in keys), keys _bs = len(next(iter(batch.values()))) assert len(keys) == _bs, f"{len(keys)} != {_bs}" - # batch = {k: v for k, v in batch.items() if not k.endswith("_lengths")} - - logging.info("decoding, utt_id: {}".format(keys)) - # N-best list of (text, token, token_int, hyp_object) - time_beg = time.time() + vad_results = speech2vadsegment(**batch) - time_end = time.time() fbanks, vadsegments = vad_results[0], vad_results[1] for i, segments in enumerate(vadsegments): - result_segments = [["", [], [], ]] + result_segments = [["", [], [], []]] for j, segment_idx in enumerate(segments): bed_idx, end_idx = int(segment_idx[0] / 10), int(segment_idx[1] / 10) segment = fbanks[:, bed_idx:end_idx, :].to(device) @@ -600,76 +591,51 @@ def inference_modelscope( "end_time": vadsegments[i][j][1]} results = speech2text(**batch) if len(results) < 1: - hyp = Hypothesis(score=0.0, scores={}, states={}, yseq=[]) - results = [[" ", ["sil"], [2], 0, 1, 6]] * nbest - time_end = time.time() - forward_time = time_end - time_beg - lfr_factor = results[0][-1] - length = results[0][-2] - forward_time_total += forward_time - length_total += length - logging.info( - "decoding, feature length: {}, forward_time: {:.4f}, rtf: {:.4f}". - format(length, forward_time, 100 * forward_time / (length * lfr_factor))) + continue + result_cur = [results[0][:-2]] if j == 0: result_segments = result_cur else: result_segments = [[result_segments[0][i] + result_cur[0][i] for i in range(len(result_cur[0]))]] - + key = keys[0] result = result_segments[0] text, token, token_int = result[0], result[1], result[2] time_stamp = None if len(result) < 4 else result[3] - - # Create a directory: outdir/{n}best_recog + + postprocessed_result = postprocess_utils.sentence_postprocess(token, time_stamp) + text_postprocessed = "" + time_stamp_postprocessed = "" + text_postprocessed_punc = postprocessed_result + if len(postprocessed_result) == 3: + text_postprocessed, time_stamp_postprocessed, word_lists = postprocessed_result[0], \ + postprocessed_result[1], \ + postprocessed_result[2] + text_postprocessed_punc = "" + if len(word_lists) > 0 and text2punc is not None: + text_postprocessed_punc, punc_id_list = text2punc(word_lists, 20) + + item = {'key': key, 'value': text_postprocessed_punc} + if text_postprocessed != "": + item['text_postprocessed'] = text_postprocessed + if time_stamp_postprocessed != "": + item['time_stamp'] = time_stamp_postprocessed + + asr_result_list.append(item) + finish_count += 1 + # asr_utils.print_progress(finish_count / file_count) if writer is not None: # Write the result to each file ibest_writer["token"][key] = " ".join(token) ibest_writer["token_int"][key] = " ".join(map(str, token_int)) ibest_writer["vad"][key] = "{}".format(vadsegments) - - if text is not None: - postprocessed_result = postprocess_utils.sentence_postprocess(token, time_stamp) - if len(postprocessed_result) == 3: - text_postprocessed, time_stamp_postprocessed, word_lists = postprocessed_result[0], \ - postprocessed_result[1], \ - postprocessed_result[2] - if len(word_lists) > 0: - text_postprocessed_punc, punc_id_list = text2punc(word_lists, 20) - text_postprocessed_punc_time_stamp = json.dumps({"predictions": text_postprocessed_punc, - "time_stamp": time_stamp_postprocessed}, - ensure_ascii=False) - else: - text_postprocessed_punc = "" - punc_id_list = [] - text_postprocessed_punc_time_stamp = "" - - else: - text_postprocessed = "" - time_stamp_postprocessed = "" - word_lists = "" - text_postprocessed_punc_time_stamp = "" - punc_id_list = "" - text_postprocessed_punc = "" - - item = {'key': key, 'value': text_postprocessed_punc, 'text_postprocessed': text_postprocessed, - 'time_stamp': time_stamp_postprocessed, 'token': token} - asr_result_list.append(item) - finish_count += 1 - # asr_utils.print_progress(finish_count / file_count) - if writer is not None: - ibest_writer["text"][key] = text_postprocessed - ibest_writer["punc_id"][key] = "{}".format(punc_id_list) - ibest_writer["text_with_punc"][key] = text_postprocessed_punc_time_stamp - if time_stamp_postprocessed is not None: - ibest_writer["time_stamp"][key] = "{}".format(time_stamp_postprocessed) - - logging.info("decoding, utt: {}, predictions: {}, time_stamp: {}".format(key, text_postprocessed_punc, - time_stamp_postprocessed)) - - logging.info("decoding, feature length total: {}, forward_time total: {:.4f}, rtf avg: {:.4f}". - format(length_total, forward_time_total, 100 * forward_time_total / (length_total * lfr_factor+1e-6))) + ibest_writer["text"][key] = text_postprocessed + ibest_writer["text_with_punc"][key] = text_postprocessed_punc + if time_stamp_postprocessed is not None: + ibest_writer["time_stamp"][key] = "{}".format(time_stamp_postprocessed) + + logging.info("decoding, utt: {}, predictions: {}".format(key, text_postprocessed_punc)) return asr_result_list return _forward