diff --git a/runtime/onnxruntime/bin/funasr-onnx-offline-rtf.cpp b/runtime/onnxruntime/bin/funasr-onnx-offline-rtf.cpp index 39928b4cd..b1a7c870c 100644 --- a/runtime/onnxruntime/bin/funasr-onnx-offline-rtf.cpp +++ b/runtime/onnxruntime/bin/funasr-onnx-offline-rtf.cpp @@ -83,6 +83,10 @@ void runReg(FUNASR_HANDLE asr_handle, vector wav_list, vector wa if(stamp !=""){ LOG(INFO) << "Thread: " << this_thread::get_id() << "," << wav_ids[i] << " : " << stamp; } + string stamp_sents = FunASRGetStampSents(result); + if(stamp_sents !=""){ + LOG(INFO)<< wav_ids[i] <<" : "<msg = msg_itn; } #endif - + if (!(p_result->stamp).empty()){ + p_result->stamp_sents = funasr::TimestampSentence(p_result->msg, p_result->stamp); + } return p_result; } @@ -399,6 +401,9 @@ p_result->msg = msg_itn; } #endif + if (!(p_result->stamp).empty()){ + p_result->stamp_sents = funasr::TimestampSentence(p_result->msg, p_result->stamp); + } return p_result; } @@ -546,7 +551,9 @@ p_result->tpass_msg = msg_itn; } #endif - + if (!(p_result->stamp).empty()){ + p_result->stamp_sents = funasr::TimestampSentence(p_result->tpass_msg, p_result->stamp); + } if(frame != NULL){ delete frame; frame = NULL; @@ -603,6 +610,15 @@ return p_result->stamp.c_str(); } + _FUNASRAPI const char* FunASRGetStampSents(FUNASR_RESULT result) + { + funasr::FUNASR_RECOG_RESULT * p_result = (funasr::FUNASR_RECOG_RESULT*)result; + if(!p_result) + return nullptr; + + return p_result->stamp_sents.c_str(); + } + _FUNASRAPI const char* FunASRGetTpassResult(FUNASR_RESULT result,int n_index) { funasr::FUNASR_RECOG_RESULT * p_result = (funasr::FUNASR_RECOG_RESULT*)result; diff --git a/runtime/onnxruntime/src/util.cpp b/runtime/onnxruntime/src/util.cpp index 2738d35ec..ac793f521 100644 --- a/runtime/onnxruntime/src/util.cpp +++ b/runtime/onnxruntime/src/util.cpp @@ -255,7 +255,8 @@ void TimestampAdd(std::deque &alignment_str1, std::string str_word){ } bool TimestampIsPunctuation(const std::string& str) { - const std::string punctuation = u8",。?、,.?"; + const std::string punctuation = u8",。?、,?"; + // const std::string punctuation = u8",。?、,.?"; for (char ch : str) { if (punctuation.find(ch) == std::string::npos) { return false; @@ -557,6 +558,76 @@ std::string TimestampSmooth(std::string &text, std::string &text_itn, std::strin return timestamps_str; } +std::string TimestampSentence(std::string &text, std::string &str_time){ + std::vector characters; + funasr::TimestampSplitChiEngCharacters(text, characters); + vector> timestamps = funasr::ParseTimestamps(str_time); + + int idx_str = 0, idx_ts = 0; + int start = -1, end = -1; + std::string text_seg = ""; + std::string ts_sentences = ""; + std::string ts_sent = ""; + vector> ts_seg; + while(idx_str < characters.size()){ + if (TimestampIsPunctuation(characters[idx_str])){ + if(ts_seg.size() >0){ + if (ts_seg[0].size() == 2){ + start = ts_seg[0][0]; + } + if (ts_seg[ts_seg.size()-1].size() == 2){ + end = ts_seg[ts_seg.size()-1][1]; + } + } + // format + ts_sent += "{'text':'" + text_seg + "',"; + ts_sent += "'start':'" + to_string(start) + "',"; + ts_sent += "'end':'" + to_string(end) + "',"; + ts_sent += "'ts_list':" + VectorToString(ts_seg) + "}"; + + if (idx_str == characters.size()-1){ + ts_sentences += ts_sent; + } else{ + ts_sentences += ts_sent + ","; + } + + // clear + idx_str++; + text_seg = ""; + ts_sent = ""; + start = 0; + end = 0; + ts_seg.clear(); + } else if(idx_ts < timestamps.size()) { + if (text_seg.empty()){ + text_seg = characters[idx_str]; + }else{ + text_seg += " " + characters[idx_str]; + } + ts_seg.push_back(timestamps[idx_ts]); + idx_str++; + idx_ts++; + } + } + // for none punc results + if(ts_seg.size() >0){ + if (ts_seg[0].size() == 2){ + start = ts_seg[0][0]; + } + if (ts_seg[ts_seg.size()-1].size() == 2){ + end = ts_seg[ts_seg.size()-1][1]; + } + // format + ts_sent += "{'text':'" + text_seg + "',"; + ts_sent += "'start':'" + to_string(start) + "',"; + ts_sent += "'end':'" + to_string(end) + "',"; + ts_sent += "'ts_list':" + VectorToString(ts_seg) + "}"; + ts_sentences += ts_sent; + } + + return "[" +ts_sentences + "]"; +} + std::vector split(const std::string &s, char delim) { std::vector elems; std::stringstream ss(s); diff --git a/runtime/onnxruntime/src/util.h b/runtime/onnxruntime/src/util.h index 46d24b3bc..eda9b49f2 100644 --- a/runtime/onnxruntime/src/util.h +++ b/runtime/onnxruntime/src/util.h @@ -47,7 +47,7 @@ void TimestampSplitChiEngCharacters(const std::string &input_str, std::vector &characters); std::string VectorToString(const std::vector>& vec); std::string TimestampSmooth(std::string &text, std::string &text_itn, std::string &str_time); - +std::string TimestampSentence(std::string &text, std::string &str_time); std::vector split(const std::string &s, char delim); template diff --git a/runtime/websocket/bin/websocket-server-2pass.cpp b/runtime/websocket/bin/websocket-server-2pass.cpp index 499c95087..44dd82e5e 100644 --- a/runtime/websocket/bin/websocket-server-2pass.cpp +++ b/runtime/websocket/bin/websocket-server-2pass.cpp @@ -80,6 +80,12 @@ nlohmann::json handle_result(FUNASR_RESULT result) { jsonresult["timestamp"] = tmp_stamp_msg; } + std::string tmp_stamp_sents = FunASRGetStampSents(result); + if (tmp_stamp_sents != "") { + LOG(INFO) << "offline stamp_sents : " << tmp_stamp_sents; + jsonresult["stamp_sents"] = tmp_stamp_sents; + } + return jsonresult; } // feed buffer to asr engine for decoder @@ -318,7 +324,7 @@ void WebSocketServer::check_and_clean_connection() { data_msg->msg["is_eof"]=true; guard_decoder.unlock(); to_remove.push_back(hdl); - LOG(INFO)<<"connection is closed: "<& buffer, if (!buffer.empty() && hotwords_embedding.size() > 0) { std::string asr_result; std::string stamp_res; + std::string stamp_sents; try{ FUNASR_RESULT Result = FunOfflineInferBuffer( asr_handle, buffer.data(), buffer.size(), RASR_NONE, NULL, @@ -81,6 +82,7 @@ void WebSocketServer::do_decoder(const std::vector& buffer, asr_result = ((FUNASR_RECOG_RESULT*)Result)->msg; // get decode result stamp_res = ((FUNASR_RECOG_RESULT*)Result)->stamp; + stamp_sents = ((FUNASR_RECOG_RESULT*)Result)->stamp_sents; FunASRFreeResult(Result); }catch (std::exception const& e) { LOG(ERROR) << e.what(); @@ -95,6 +97,9 @@ void WebSocketServer::do_decoder(const std::vector& buffer, if(stamp_res != ""){ jsonresult["timestamp"] = stamp_res; } + if(stamp_sents != ""){ + jsonresult["stamp_sents"] = stamp_sents; + } jsonresult["wav_name"] = wav_name; // send the json to client @@ -227,7 +232,7 @@ void WebSocketServer::check_and_clean_connection() { data_msg->msg["is_eof"]=true; guard_decoder.unlock(); to_remove.push_back(hdl); - LOG(INFO)<<"connection is closed: "< typedef struct { std::string msg=""; std::string stamp=""; + std::string stamp_sents; std::string tpass_msg=""; float snippet_time=0; } FUNASR_RECOG_RESULT;