From aa72e0ca5f7541d5e37877a91816f03883809ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=BE=E8=81=AA?= Date: Wed, 25 Sep 2024 23:44:42 +0800 Subject: [PATCH] add sensevoice-small --- runtime/onnxruntime/include/com-define.h | 3 + runtime/onnxruntime/include/funasrruntime.h | 3 +- runtime/onnxruntime/include/model.h | 2 + runtime/onnxruntime/include/offline-stream.h | 3 + runtime/onnxruntime/src/sensevoice-small.cpp | 377 +++++++++++++++++++ runtime/onnxruntime/src/sensevoice-small.h | 116 ++++++ 6 files changed, 503 insertions(+), 1 deletion(-) create mode 100644 runtime/onnxruntime/src/sensevoice-small.cpp create mode 100644 runtime/onnxruntime/src/sensevoice-small.h diff --git a/runtime/onnxruntime/include/com-define.h b/runtime/onnxruntime/include/com-define.h index 5f71f7b3c..6b4c08f0d 100644 --- a/runtime/onnxruntime/include/com-define.h +++ b/runtime/onnxruntime/include/com-define.h @@ -36,6 +36,9 @@ namespace funasr { #define HOTWORD_SEP " " #define AUDIO_FS "audio-fs" +#define MODEL_PARA "Paraformer" +#define MODEL_SVS "SenseVoiceSmall" + // #define VAD_MODEL_PATH "vad-model" // #define VAD_CMVN_PATH "vad-cmvn" // #define VAD_CONFIG_PATH "vad-config" diff --git a/runtime/onnxruntime/include/funasrruntime.h b/runtime/onnxruntime/include/funasrruntime.h index ac9a3bd3a..5dedaf7ec 100644 --- a/runtime/onnxruntime/include/funasrruntime.h +++ b/runtime/onnxruntime/include/funasrruntime.h @@ -101,7 +101,8 @@ _FUNASRAPI void FunOfflineReset(FUNASR_HANDLE handle, FUNASR_DEC_HANDLE // buffer _FUNASRAPI FUNASR_RESULT FunOfflineInferBuffer(FUNASR_HANDLE handle, const char* sz_buf, int n_len, FUNASR_MODE mode, QM_CALLBACK fn_callback, const std::vector> &hw_emb, - int sampling_rate=16000, std::string wav_format="pcm", bool itn=true, FUNASR_DEC_HANDLE dec_handle=nullptr); + int sampling_rate=16000, std::string wav_format="pcm", bool itn=true, FUNASR_DEC_HANDLE dec_handle=nullptr, + std::string svs_lang="auto", bool svs_itn=true); // file, support wav & pcm _FUNASRAPI FUNASR_RESULT FunOfflineInfer(FUNASR_HANDLE handle, const char* sz_filename, FUNASR_MODE mode, QM_CALLBACK fn_callback, const std::vector> &hw_emb, diff --git a/runtime/onnxruntime/include/model.h b/runtime/onnxruntime/include/model.h index 1064c4c0f..a49baebd0 100644 --- a/runtime/onnxruntime/include/model.h +++ b/runtime/onnxruntime/include/model.h @@ -24,6 +24,8 @@ class Model { virtual std::string Forward(float *din, int len, bool input_finished, const std::vector> &hw_emb={{0.0}}, void* wfst_decoder=nullptr){return "";}; virtual std::vector Forward(float** din, int* len, bool input_finished, const std::vector> &hw_emb={{0.0}}, void* wfst_decoder=nullptr, int batch_in=1) {return std::vector();}; + virtual std::vector Forward(float** din, int* len, bool input_finished, std::string svs_lang="auto", bool svs_itn=false, int batch_in=1) + {return std::vector();}; virtual std::string Rescoring() = 0; virtual void InitHwCompiler(const std::string &hw_model, int thread_num){}; virtual void InitSegDict(const std::string &seg_dict_model){}; diff --git a/runtime/onnxruntime/include/offline-stream.h b/runtime/onnxruntime/include/offline-stream.h index cc0f1c43b..10bd6df70 100644 --- a/runtime/onnxruntime/include/offline-stream.h +++ b/runtime/onnxruntime/include/offline-stream.h @@ -9,6 +9,7 @@ #include "vad-model.h" #if !defined(__APPLE__) #include "itn-model.h" +#include "com-define.h" #endif namespace funasr { @@ -26,11 +27,13 @@ class OfflineStream { bool UseVad(){return use_vad;}; bool UsePunc(){return use_punc;}; bool UseITN(){return use_itn;}; + std::string GetModelType(){return model_type;}; private: bool use_vad=false; bool use_punc=false; bool use_itn=false; + std::string model_type = MODEL_PARA; }; OfflineStream *CreateOfflineStream(std::map& model_path, int thread_num=1, bool use_gpu=false, int batch_size=1); diff --git a/runtime/onnxruntime/src/sensevoice-small.cpp b/runtime/onnxruntime/src/sensevoice-small.cpp new file mode 100644 index 000000000..10eb907eb --- /dev/null +++ b/runtime/onnxruntime/src/sensevoice-small.cpp @@ -0,0 +1,377 @@ +/** + * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. + * MIT License (https://opensource.org/licenses/MIT) +*/ + +#include "precomp.h" +#include "sensevoice-small.h" +#include + +using namespace std; +namespace funasr { + +SenseVoiceSmall::SenseVoiceSmall() +:use_hotword(false), + env_(ORT_LOGGING_LEVEL_ERROR, "sensevoice"),session_options_{} { +} + +// offline +void SenseVoiceSmall::InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num){ + LoadConfigFromYaml(am_config.c_str()); + // knf options + fbank_opts_.frame_opts.dither = 0; + fbank_opts_.mel_opts.num_bins = n_mels; + fbank_opts_.frame_opts.samp_freq = asr_sample_rate; + fbank_opts_.frame_opts.window_type = window_type; + fbank_opts_.frame_opts.frame_shift_ms = frame_shift; + fbank_opts_.frame_opts.frame_length_ms = frame_length; + fbank_opts_.energy_floor = 0; + fbank_opts_.mel_opts.debug_mel = false; + + // session_options_.SetInterOpNumThreads(1); + session_options_.SetIntraOpNumThreads(thread_num); + session_options_.SetGraphOptimizationLevel(ORT_ENABLE_ALL); + // DisableCpuMemArena can improve performance + session_options_.DisableCpuMemArena(); + + try { + m_session_ = std::make_unique(env_, ORTSTRING(am_model).c_str(), session_options_); + LOG(INFO) << "Successfully load model from " << am_model; + } catch (std::exception const &e) { + LOG(ERROR) << "Error when load am onnx model: " << e.what(); + exit(-1); + } + + string strName; + GetInputName(m_session_.get(), strName); + m_strInputNames.push_back(strName.c_str()); + GetInputName(m_session_.get(), strName,1); + m_strInputNames.push_back(strName); + GetInputName(m_session_.get(), strName,2); + m_strInputNames.push_back(strName); + GetInputName(m_session_.get(), strName,3); + m_strInputNames.push_back(strName); + + size_t numOutputNodes = m_session_->GetOutputCount(); + for(int index=0; indexwindow_type = frontend_conf["window"].as(); + this->n_mels = frontend_conf["n_mels"].as(); + this->frame_length = frontend_conf["frame_length"].as(); + this->frame_shift = frontend_conf["frame_shift"].as(); + this->lfr_m = frontend_conf["lfr_m"].as(); + this->lfr_n = frontend_conf["lfr_n"].as(); + + this->encoder_size = encoder_conf["output_size"].as(); + this->fsmn_dims = encoder_conf["output_size"].as(); + + this->asr_sample_rate = frontend_conf["fs"].as(); + }catch(exception const &e){ + LOG(ERROR) << "Error when load argument from vad config YAML."; + exit(-1); + } +} + +SenseVoiceSmall::~SenseVoiceSmall() +{ + if(vocab){ + delete vocab; + } + if(lm_vocab){ + delete lm_vocab; + } + if(seg_dict){ + delete seg_dict; + } + if(phone_set_){ + delete phone_set_; + } +} + +void SenseVoiceSmall::StartUtterance() +{ +} + +void SenseVoiceSmall::EndUtterance() +{ +} + +void SenseVoiceSmall::Reset() +{ +} + +void SenseVoiceSmall::FbankKaldi(float sample_rate, const float* waves, int len, std::vector> &asr_feats) { + knf::OnlineFbank fbank_(fbank_opts_); + std::vector buf(len); + for (int32_t i = 0; i != len; ++i) { + buf[i] = waves[i] * 32768; + } + fbank_.AcceptWaveform(sample_rate, buf.data(), buf.size()); + + int32_t frames = fbank_.NumFramesReady(); + for (int32_t i = 0; i != frames; ++i) { + const float *frame = fbank_.GetFrame(i); + std::vector frame_vector(frame, frame + fbank_opts_.mel_opts.num_bins); + asr_feats.emplace_back(frame_vector); + } +} + +void SenseVoiceSmall::LoadCmvn(const char *filename) +{ + ifstream cmvn_stream(filename); + if (!cmvn_stream.is_open()) { + LOG(ERROR) << "Failed to open file: " << filename; + exit(-1); + } + string line; + + while (getline(cmvn_stream, line)) { + istringstream iss(line); + vector line_item{istream_iterator{iss}, istream_iterator{}}; + if (line_item[0] == "") { + getline(cmvn_stream, line); + istringstream means_lines_stream(line); + vector means_lines{istream_iterator{means_lines_stream}, istream_iterator{}}; + if (means_lines[0] == "") { + for (int j = 3; j < means_lines.size() - 1; j++) { + means_list_.push_back(stof(means_lines[j])); + } + continue; + } + } + else if (line_item[0] == "") { + getline(cmvn_stream, line); + istringstream vars_lines_stream(line); + vector vars_lines{istream_iterator{vars_lines_stream}, istream_iterator{}}; + if (vars_lines[0] == "") { + for (int j = 3; j < vars_lines.size() - 1; j++) { + vars_list_.push_back(stof(vars_lines[j])*scale); + } + continue; + } + } + } +} + +string SenseVoiceSmall::CTCSearch(float * in, std::vector paraformer_length, std::vector outputShape) +{ + std::string unicodeChar = "▁"; + int32_t vocab_size = outputShape[2]; + + std::vector tokens; + std::string text=""; + int32_t prev_id = -1; + for (int32_t t = 0; t != paraformer_length[0]; ++t) { + auto y = std::distance( + static_cast(in), + std::max_element( + static_cast(in), + static_cast(in) + vocab_size)); + in += vocab_size; + + if (y != blank_id && y != prev_id) { + tokens.push_back(y); + } + prev_id = y; + } + string str_lang = ""; + string str_emo = ""; + string str_event = ""; + string str_itn = ""; + if(tokens.size() >=3){ + str_lang = vocab->Id2String(tokens[0]); + str_emo = vocab->Id2String(tokens[1]); + str_event = vocab->Id2String(tokens[2]); + str_itn = vocab->Id2String(tokens[3]); + } + + for(int32_t i = 4; i < tokens.size(); ++i){ + string word = vocab->Id2String(tokens[i]); + size_t found = word.find(unicodeChar); + if(found != std::string::npos){ + text += " " + word.substr(3); + }else{ + text += word; + } + } + if(str_itn == "<|withitn|>"){ + if(str_lang == "<|zh|>"){ + text += "。"; + }else{ + text += "."; + } + } + + return str_lang + str_emo + str_event + " " + text; +} + +void SenseVoiceSmall::LfrCmvn(std::vector> &asr_feats) { + + std::vector> out_feats; + int T = asr_feats.size(); + int T_lrf = ceil(1.0 * T / lfr_n); + + // Pad frames at start(copy first frame) + for (int i = 0; i < (lfr_m - 1) / 2; i++) { + asr_feats.insert(asr_feats.begin(), asr_feats[0]); + } + // Merge lfr_m frames as one,lfr_n frames per window + T = T + (lfr_m - 1) / 2; + std::vector p; + for (int i = 0; i < T_lrf; i++) { + if (lfr_m <= T - i * lfr_n) { + for (int j = 0; j < lfr_m; j++) { + p.insert(p.end(), asr_feats[i * lfr_n + j].begin(), asr_feats[i * lfr_n + j].end()); + } + out_feats.emplace_back(p); + p.clear(); + } else { + // Fill to lfr_m frames at last window if less than lfr_m frames (copy last frame) + int num_padding = lfr_m - (T - i * lfr_n); + for (int j = 0; j < (asr_feats.size() - i * lfr_n); j++) { + p.insert(p.end(), asr_feats[i * lfr_n + j].begin(), asr_feats[i * lfr_n + j].end()); + } + for (int j = 0; j < num_padding; j++) { + p.insert(p.end(), asr_feats[asr_feats.size() - 1].begin(), asr_feats[asr_feats.size() - 1].end()); + } + out_feats.emplace_back(p); + p.clear(); + } + } + // Apply cmvn + for (auto &out_feat: out_feats) { + for (int j = 0; j < means_list_.size(); j++) { + out_feat[j] = (out_feat[j] + means_list_[j]) * vars_list_[j]; + } + } + asr_feats = out_feats; +} + +std::vector> SenseVoiceSmall::CompileHotwordEmbedding(std::string &hotwords) { + int embedding_dim = encoder_size; + std::vector> hw_emb; + std::vector vec(embedding_dim, 0); + hw_emb.push_back(vec); + return hw_emb; +} + +std::vector SenseVoiceSmall::Forward(float** din, int* len, bool input_finished, std::string svs_lang, bool svs_itn, int batch_in) +{ + std::vector results; + string result=""; + int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins; + + if(batch_in != 1){ + results.push_back(result); + return results; + } + + std::vector> asr_feats; + FbankKaldi(asr_sample_rate, din[0], len[0], asr_feats); + if(asr_feats.size() == 0){ + results.push_back(result); + return results; + } + LfrCmvn(asr_feats); + int32_t feat_dim = lfr_m*in_feat_dim; + int32_t num_frames = asr_feats.size(); + + std::vector wav_feats; + for (const auto &frame_feat: asr_feats) { + wav_feats.insert(wav_feats.end(), frame_feat.begin(), frame_feat.end()); + } + + //lid textnorm + int svs_lid = 0; + int svs_itnid = 15; + if(lid_map.find(svs_lang) != lid_map.end()){ + svs_lid = lid_map[svs_lang]; + } + if(svs_itn){ + svs_itnid = 14; + } + +#ifdef _WIN_X86 + Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); +#else + Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); +#endif + + const int64_t input_shape_[3] = {1, num_frames, feat_dim}; + Ort::Value onnx_feats = Ort::Value::CreateTensor(m_memoryInfo, + wav_feats.data(), + wav_feats.size(), + input_shape_, + 3); + + const int64_t paraformer_length_shape[1] = {1}; + std::vector paraformer_length; + paraformer_length.emplace_back(num_frames); + Ort::Value onnx_feats_len = Ort::Value::CreateTensor( + m_memoryInfo, paraformer_length.data(), paraformer_length.size(), paraformer_length_shape, 1); + + const int64_t lid_shape[1] = {1}; + std::vector lid_length; + lid_length.emplace_back(svs_lid); + Ort::Value onnx_lid = Ort::Value::CreateTensor( + m_memoryInfo, lid_length.data(), lid_length.size(), lid_shape, 1); + + const int64_t textnorm_shape[1] = {1}; + std::vector textnorm_length; + textnorm_length.emplace_back(svs_itnid); + Ort::Value onnx_itn = Ort::Value::CreateTensor( + m_memoryInfo, textnorm_length.data(), textnorm_length.size(), textnorm_shape, 1); + + std::vector input_onnx; + input_onnx.emplace_back(std::move(onnx_feats)); + input_onnx.emplace_back(std::move(onnx_feats_len)); + input_onnx.emplace_back(std::move(onnx_lid)); + input_onnx.emplace_back(std::move(onnx_itn)); + + try { + auto outputTensor = m_session_->Run(Ort::RunOptions{nullptr}, m_szInputNames.data(), input_onnx.data(), input_onnx.size(), m_szOutputNames.data(), m_szOutputNames.size()); + float* floatData = outputTensor[0].GetTensorMutableData(); + std::vector outputShape = outputTensor[0].GetTensorTypeAndShapeInfo().GetShape(); + + result = CTCSearch(floatData, paraformer_length, outputShape); + } + catch (std::exception const &e) + { + LOG(ERROR)<> &asr_feats); + + std::shared_ptr hw_m_session = nullptr; + Ort::Env hw_env_; + Ort::SessionOptions hw_session_options; + vector hw_m_strInputNames, hw_m_strOutputNames; + vector hw_m_szInputNames; + vector hw_m_szOutputNames; + bool use_hotword; + + public: + SenseVoiceSmall(); + ~SenseVoiceSmall(); + void InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num); + // online + // void InitAsr(const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num); + // 2pass + // void InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num); + // void InitHwCompiler(const std::string &hw_model, int thread_num); + // void InitSegDict(const std::string &seg_dict_model); + std::vector> CompileHotwordEmbedding(std::string &hotwords); + void Reset(); + void FbankKaldi(float sample_rate, const float* waves, int len, std::vector> &asr_feats); + std::vector Forward(float** din, int* len, bool input_finished=true, std::string svs_lang="auto", bool svs_itn=true, int batch_in=1); + string CTCSearch( float * in, std::vector paraformer_length, std::vector outputShape); + + string Rescoring(); + string GetLang(){return language;}; + int GetAsrSampleRate() { return asr_sample_rate; }; + int GetBatchSize() {return batch_size_;}; + void StartUtterance(); + void EndUtterance(); + // void InitLm(const std::string &lm_file, const std::string &lm_cfg_file, const std::string &lex_file); + // string BeamSearch(WfstDecoder* &wfst_decoder, float* in, int n_len, int64_t token_nums); + // string FinalizeDecode(WfstDecoder* &wfst_decoder, + // bool is_stamp=false, std::vector us_alphas={0}, std::vector us_cif_peak={0}); + // Vocab* GetVocab(); + // Vocab* GetLmVocab(); + // PhoneSet* GetPhoneSet(); + + knf::FbankOptions fbank_opts_; + vector means_list_; + vector vars_list_; + int lfr_m = PARA_LFR_M; + int lfr_n = PARA_LFR_N; + + // paraformer-offline + std::shared_ptr m_session_ = nullptr; + Ort::Env env_; + Ort::SessionOptions session_options_; + + vector m_strInputNames, m_strOutputNames; + vector m_szInputNames; + vector m_szOutputNames; + + std::string language="zh-cn"; + + // paraformer-online + std::shared_ptr encoder_session_ = nullptr; + std::shared_ptr decoder_session_ = nullptr; + vector en_strInputNames, en_strOutputNames; + vector en_szInputNames_; + vector en_szOutputNames_; + vector de_strInputNames, de_strOutputNames; + vector de_szInputNames_; + vector de_szOutputNames_; + + // lm + std::shared_ptr> lm_ = nullptr; + + string window_type = "hamming"; + int frame_length = 25; + int frame_shift = 10; + int n_mels = 80; + int encoder_size = 512; + int fsmn_layers = 16; + int fsmn_lorder = 10; + int fsmn_dims = 512; + int asr_sample_rate = MODEL_SAMPLE_RATE; + int batch_size_ = 1; + int blank_id = 0; + //dict + std::map lid_map = { + {"auto", 0}, + {"zh", 3}, + {"en", 4}, + {"yue", 7}, + {"ja", 11}, + {"ko", 12}, + {"nospeech", 13} + }; + + }; + +} // namespace funasr