speech_recognition/FunASR
1
0
Fork 0
mirror of https://github.com/modelscope/FunASR synced 2025-09-15 14:48:36 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix compute cer problems

Browse Source
This commit is contained in:
北念 2023-04-04 14:26:22 +08:00
parent cbbb300743
commit cf843d144a
20 changed files with 27 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
egs/aishell/transformer/utils/compute_wer.py
View File

@ -45,8 +45,8 @@ def compute_wer(ref_file,
if out_item['wrong'] > 0: if out_item['wrong'] > 0:
rst['wrong_sentences'] += 1 rst['wrong_sentences'] += 1
cer_detail_writer.write(hyp_key + print_cer_detail(out_item) + '\n') cer_detail_writer.write(hyp_key + print_cer_detail(out_item) + '\n')
cer_detail_writer.write("ref:" + '\t' + "".join(ref_dict[hyp_key]) + '\n') cer_detail_writer.write("ref:" + '\t' + " ".join(list(map(lambda x: x.lower(), ref_dict[hyp_key]))) + '\n')
cer_detail_writer.write("hyp:" + '\t' + "".join(hyp_dict[hyp_key]) + '\n') cer_detail_writer.write("hyp:" + '\t' + " ".join(list(map(lambda x: x.lower(), hyp_dict[hyp_key]))) + '\n')
if rst['Wrd'] > 0: if rst['Wrd'] > 0:
rst['Err'] = round(rst['wrong_words'] * 100 / rst['Wrd'], 2) rst['Err'] = round(rst['wrong_words'] * 100 / rst['Wrd'], 2)

2
egs_modelscope/asr/data2vec/speech_data2vec_pretrain-paraformer-zh-cn-aishell2-16k/infer.py
View File

@ -74,7 +74,7 @@ def modelscope_infer(params):
# If text exists, compute CER # If text exists, compute CER
text_in = os.path.join(params["data_dir"], "text") text_in = os.path.join(params["data_dir"], "text")
if os.path.exists(text_in): if os.path.exists(text_in):
text_proc_file = os.path.join(best_recog_path, "token") text_proc_file = os.path.join(best_recog_path, "text")
compute_wer(text_in, text_proc_file, os.path.join(best_recog_path, "text.cer")) compute_wer(text_in, text_proc_file, os.path.join(best_recog_path, "text.cer"))

2
egs_modelscope/asr/data2vec/speech_data2vec_pretrain-paraformer-zh-cn-aishell2-16k/infer_after_finetune.py
View File

@ -38,7 +38,7 @@ def modelscope_infer_after_finetune(params):
# computer CER if GT text is set # computer CER if GT text is set
text_in = os.path.join(params["data_dir"], "text") text_in = os.path.join(params["data_dir"], "text")
if os.path.exists(text_in): if os.path.exists(text_in):
text_proc_file = os.path.join(decoding_path, "1best_recog/token") text_proc_file = os.path.join(decoding_path, "1best_recog/text")
compute_wer(text_in, text_proc_file, os.path.join(decoding_path, "text.cer")) compute_wer(text_in, text_proc_file, os.path.join(decoding_path, "text.cer"))

2
egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/infer.py
View File

@ -74,7 +74,7 @@ def modelscope_infer(params):
# If text exists, compute CER # If text exists, compute CER
text_in = os.path.join(params["data_dir"], "text") text_in = os.path.join(params["data_dir"], "text")
if os.path.exists(text_in): if os.path.exists(text_in):
text_proc_file = os.path.join(best_recog_path, "token") text_proc_file = os.path.join(best_recog_path, "text")
compute_wer(text_in, text_proc_file, os.path.join(best_recog_path, "text.cer")) compute_wer(text_in, text_proc_file, os.path.join(best_recog_path, "text.cer"))

2
egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/infer_after_finetune.py
View File

@ -38,7 +38,7 @@ def modelscope_infer_after_finetune(params):
# computer CER if GT text is set # computer CER if GT text is set
text_in = os.path.join(params["data_dir"], "text") text_in = os.path.join(params["data_dir"], "text")
if os.path.exists(text_in): if os.path.exists(text_in):
text_proc_file = os.path.join(decoding_path, "1best_recog/token") text_proc_file = os.path.join(decoding_path, "1best_recog/text")
compute_wer(text_in, text_proc_file, os.path.join(decoding_path, "text.cer")) compute_wer(text_in, text_proc_file, os.path.join(decoding_path, "text.cer"))

4
egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer.sh
View File

@ -63,8 +63,8 @@ fi
if [ $stage -le 2 ] && [ $stop_stage -ge 2 ];then if [ $stage -le 2 ] && [ $stop_stage -ge 2 ];then
echo "Computing WER ..." echo "Computing WER ..."
python utils/proce_text.py ${output_dir}/1best_recog/text ${output_dir}/1best_recog/text.proc cp ${output_dir}/1best_recog/text ${output_dir}/1best_recog/text.proc
python utils/proce_text.py ${data_dir}/text ${output_dir}/1best_recog/text.ref cp ${data_dir}/text ${output_dir}/1best_recog/text.ref
python utils/compute_wer.py ${output_dir}/1best_recog/text.ref ${output_dir}/1best_recog/text.proc ${output_dir}/1best_recog/text.cer python utils/compute_wer.py ${output_dir}/1best_recog/text.ref ${output_dir}/1best_recog/text.proc ${output_dir}/1best_recog/text.cer
tail -n 3 ${output_dir}/1best_recog/text.cer tail -n 3 ${output_dir}/1best_recog/text.cer
fi fi

2
egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer_after_finetune.py
View File

@ -34,7 +34,7 @@ def modelscope_infer_after_finetune(params):
# computer CER if GT text is set # computer CER if GT text is set
text_in = os.path.join(params["data_dir"], "text") text_in = os.path.join(params["data_dir"], "text")
if os.path.exists(text_in): if os.path.exists(text_in):
text_proc_file = os.path.join(decoding_path, "1best_recog/token") text_proc_file = os.path.join(decoding_path, "1best_recog/text")
compute_wer(text_in, text_proc_file, os.path.join(decoding_path, "text.cer")) compute_wer(text_in, text_proc_file, os.path.join(decoding_path, "text.cer"))

4
egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-8k-common-vocab8358-tensorflow1/infer.sh
View File

@ -63,8 +63,8 @@ fi
if [ $stage -le 2 ] && [ $stop_stage -ge 2 ];then if [ $stage -le 2 ] && [ $stop_stage -ge 2 ];then
echo "Computing WER ..." echo "Computing WER ..."
python utils/proce_text.py ${output_dir}/1best_recog/text ${output_dir}/1best_recog/text.proc cp ${output_dir}/1best_recog/text ${output_dir}/1best_recog/text.proc
python utils/proce_text.py ${data_dir}/text ${output_dir}/1best_recog/text.ref cp ${data_dir}/text ${output_dir}/1best_recog/text.ref
python utils/compute_wer.py ${output_dir}/1best_recog/text.ref ${output_dir}/1best_recog/text.proc ${output_dir}/1best_recog/text.cer python utils/compute_wer.py ${output_dir}/1best_recog/text.ref ${output_dir}/1best_recog/text.proc ${output_dir}/1best_recog/text.cer
tail -n 3 ${output_dir}/1best_recog/text.cer tail -n 3 ${output_dir}/1best_recog/text.cer
fi fi

2
egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-8k-common-vocab8358-tensorflow1/infer_after_finetune.py
View File

@ -34,7 +34,7 @@ def modelscope_infer_after_finetune(params):
# computer CER if GT text is set # computer CER if GT text is set
text_in = os.path.join(params["data_dir"], "text") text_in = os.path.join(params["data_dir"], "text")
if os.path.exists(text_in): if os.path.exists(text_in):
text_proc_file = os.path.join(decoding_path, "1best_recog/token") text_proc_file = os.path.join(decoding_path, "1best_recog/text")
compute_wer(text_in, text_proc_file, os.path.join(decoding_path, "text.cer")) compute_wer(text_in, text_proc_file, os.path.join(decoding_path, "text.cer"))

2
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-offline/infer.py
View File

@ -75,7 +75,7 @@ def modelscope_infer(params):
# If text exists, compute CER # If text exists, compute CER
text_in = os.path.join(params["data_dir"], "text") text_in = os.path.join(params["data_dir"], "text")
if os.path.exists(text_in): if os.path.exists(text_in):
text_proc_file = os.path.join(best_recog_path, "token") text_proc_file = os.path.join(best_recog_path, "text")
compute_wer(text_in, text_proc_file, os.path.join(best_recog_path, "text.cer")) compute_wer(text_in, text_proc_file, os.path.join(best_recog_path, "text.cer"))

2
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-offline/infer_after_finetune.py
View File

@ -39,7 +39,7 @@ def modelscope_infer_after_finetune(params):
# computer CER if GT text is set # computer CER if GT text is set
text_in = os.path.join(params["data_dir"], "text") text_in = os.path.join(params["data_dir"], "text")
if os.path.exists(text_in): if os.path.exists(text_in):
text_proc_file = os.path.join(decoding_path, "1best_recog/token") text_proc_file = os.path.join(decoding_path, "1best_recog/text")
compute_wer(text_in, text_proc_file, os.path.join(decoding_path, "text.cer")) compute_wer(text_in, text_proc_file, os.path.join(decoding_path, "text.cer"))

2
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online/infer.py
View File

@ -75,7 +75,7 @@ def modelscope_infer(params):
# If text exists, compute CER # If text exists, compute CER
text_in = os.path.join(params["data_dir"], "text") text_in = os.path.join(params["data_dir"], "text")
if os.path.exists(text_in): if os.path.exists(text_in):
text_proc_file = os.path.join(best_recog_path, "token") text_proc_file = os.path.join(best_recog_path, "text")
compute_wer(text_in, text_proc_file, os.path.join(best_recog_path, "text.cer")) compute_wer(text_in, text_proc_file, os.path.join(best_recog_path, "text.cer"))

2
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online/infer_after_finetune.py
View File

@ -39,7 +39,7 @@ def modelscope_infer_after_finetune(params):
# computer CER if GT text is set # computer CER if GT text is set
text_in = os.path.join(params["data_dir"], "text") text_in = os.path.join(params["data_dir"], "text")
if os.path.exists(text_in): if os.path.exists(text_in):
text_proc_file = os.path.join(decoding_path, "1best_recog/token") text_proc_file = os.path.join(decoding_path, "1best_recog/text")
compute_wer(text_in, text_proc_file, os.path.join(decoding_path, "text.cer")) compute_wer(text_in, text_proc_file, os.path.join(decoding_path, "text.cer"))

2
funasr/bin/asr_inference_paraformer.py
View File

@ -797,7 +797,7 @@ def inference_modelscope(
finish_count += 1 finish_count += 1
# asr_utils.print_progress(finish_count / file_count) # asr_utils.print_progress(finish_count / file_count)
if writer is not None: if writer is not None:
ibest_writer["text"][key] = text_postprocessed ibest_writer["text"][key] = " ".join(word_lists)
logging.info("decoding, utt: {}, predictions: {}".format(key, text)) logging.info("decoding, utt: {}, predictions: {}".format(key, text))
rtf_avg = "decoding, feature length total: {}, forward_time total: {:.4f}, rtf avg: {:.4f}".format(length_total, forward_time_total, 100 * forward_time_total / (length_total * lfr_factor)) rtf_avg = "decoding, feature length total: {}, forward_time total: {:.4f}, rtf avg: {:.4f}".format(length_total, forward_time_total, 100 * forward_time_total / (length_total * lfr_factor))

2
funasr/bin/asr_inference_paraformer_vad.py
View File

@ -338,7 +338,7 @@ def inference_modelscope(
ibest_writer["token"][key] = " ".join(token) ibest_writer["token"][key] = " ".join(token)
ibest_writer["token_int"][key] = " ".join(map(str, token_int)) ibest_writer["token_int"][key] = " ".join(map(str, token_int))
ibest_writer["vad"][key] = "{}".format(vadsegments) ibest_writer["vad"][key] = "{}".format(vadsegments)
ibest_writer["text"][key] = text_postprocessed ibest_writer["text"][key] = " ".join(word_lists)
ibest_writer["text_with_punc"][key] = text_postprocessed_punc ibest_writer["text_with_punc"][key] = text_postprocessed_punc
if time_stamp_postprocessed is not None: if time_stamp_postprocessed is not None:
ibest_writer["time_stamp"][key] = "{}".format(time_stamp_postprocessed) ibest_writer["time_stamp"][key] = "{}".format(time_stamp_postprocessed)

2
funasr/bin/asr_inference_paraformer_vad_punc.py
View File

@ -670,7 +670,7 @@ def inference_modelscope(
ibest_writer["token"][key] = " ".join(token) ibest_writer["token"][key] = " ".join(token)
ibest_writer["token_int"][key] = " ".join(map(str, token_int)) ibest_writer["token_int"][key] = " ".join(map(str, token_int))
ibest_writer["vad"][key] = "{}".format(vadsegments) ibest_writer["vad"][key] = "{}".format(vadsegments)
ibest_writer["text"][key] = text_postprocessed ibest_writer["text"][key] = " ".join(word_lists)
ibest_writer["text_with_punc"][key] = text_postprocessed_punc ibest_writer["text_with_punc"][key] = text_postprocessed_punc
if time_stamp_postprocessed is not None: if time_stamp_postprocessed is not None:
ibest_writer["time_stamp"][key] = "{}".format(time_stamp_postprocessed) ibest_writer["time_stamp"][key] = "{}".format(time_stamp_postprocessed)

4
funasr/bin/asr_inference_rnnt.py
View File

@ -738,13 +738,13 @@ def inference_modelscope(
ibest_writer["rtf"][key] = rtf_cur ibest_writer["rtf"][key] = rtf_cur
if text is not None: if text is not None:
text_postprocessed, _ = postprocess_utils.sentence_postprocess(token) text_postprocessed, word_lists = postprocess_utils.sentence_postprocess(token)
item = {'key': key, 'value': text_postprocessed} item = {'key': key, 'value': text_postprocessed}
asr_result_list.append(item) asr_result_list.append(item)
finish_count += 1 finish_count += 1
# asr_utils.print_progress(finish_count / file_count) # asr_utils.print_progress(finish_count / file_count)
if writer is not None: if writer is not None:
ibest_writer["text"][key] = text_postprocessed ibest_writer["text"][key] = " ".join(word_lists)
logging.info("decoding, utt: {}, predictions: {}".format(key, text)) logging.info("decoding, utt: {}, predictions: {}".format(key, text))
rtf_avg = "decoding, feature length total: {}, forward_time total: {:.4f}, rtf avg: {:.4f}".format(length_total, forward_time_total, 100 * forward_time_total / (length_total * lfr_factor)) rtf_avg = "decoding, feature length total: {}, forward_time total: {:.4f}, rtf avg: {:.4f}".format(length_total, forward_time_total, 100 * forward_time_total / (length_total * lfr_factor))

4
funasr/bin/asr_inference_uniasr.py
View File

@ -507,13 +507,13 @@ def inference_modelscope(
ibest_writer["score"][key] = str(hyp.score) ibest_writer["score"][key] = str(hyp.score)
if text is not None: if text is not None:
text_postprocessed, _ = postprocess_utils.sentence_postprocess(token) text_postprocessed, word_lists = postprocess_utils.sentence_postprocess(token)
item = {'key': key, 'value': text_postprocessed} item = {'key': key, 'value': text_postprocessed}
asr_result_list.append(item) asr_result_list.append(item)
finish_count += 1 finish_count += 1
asr_utils.print_progress(finish_count / file_count) asr_utils.print_progress(finish_count / file_count)
if writer is not None: if writer is not None:
ibest_writer["text"][key] = text_postprocessed ibest_writer["text"][key] = " ".join(word_lists)
return asr_result_list return asr_result_list
return _forward return _forward

4
funasr/bin/asr_inference_uniasr_vad.py
View File

@ -507,13 +507,13 @@ def inference_modelscope(
ibest_writer["score"][key] = str(hyp.score) ibest_writer["score"][key] = str(hyp.score)
if text is not None: if text is not None:
text_postprocessed, _ = postprocess_utils.sentence_postprocess(token) text_postprocessed, word_lists = postprocess_utils.sentence_postprocess(token)
item = {'key': key, 'value': text_postprocessed} item = {'key': key, 'value': text_postprocessed}
asr_result_list.append(item) asr_result_list.append(item)
finish_count += 1 finish_count += 1
asr_utils.print_progress(finish_count / file_count) asr_utils.print_progress(finish_count / file_count)
if writer is not None: if writer is not None:
ibest_writer["text"][key] = text_postprocessed ibest_writer["text"][key] = " ".join(word_lists)
return asr_result_list return asr_result_list
return _forward return _forward

4
funasr/utils/compute_wer.py
View File

@ -45,8 +45,8 @@ def compute_wer(ref_file,
if out_item['wrong'] > 0: if out_item['wrong'] > 0:
rst['wrong_sentences'] += 1 rst['wrong_sentences'] += 1
cer_detail_writer.write(hyp_key + print_cer_detail(out_item) + '\n') cer_detail_writer.write(hyp_key + print_cer_detail(out_item) + '\n')
cer_detail_writer.write("ref:" + '\t' + "".join(ref_dict[hyp_key]) + '\n') cer_detail_writer.write("ref:" + '\t' + " ".join(list(map(lambda x: x.lower(), ref_dict[hyp_key]))) + '\n')
cer_detail_writer.write("hyp:" + '\t' + "".join(hyp_dict[hyp_key]) + '\n') cer_detail_writer.write("hyp:" + '\t' + " ".join(list(map(lambda x: x.lower(), hyp_dict[hyp_key]))) + '\n')
if rst['Wrd'] > 0: if rst['Wrd'] > 0:
rst['Err'] = round(rst['wrong_words'] * 100 / rst['Wrd'], 2) rst['Err'] = round(rst['wrong_words'] * 100 / rst['Wrd'], 2)