diff --git a/CMakeLists.txt b/CMakeLists.txt index 52496cb..c15a220 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,17 +15,32 @@ set(COMPONENT_REQUIRES register_component() if(IDF_TARGET STREQUAL "esp32") target_link_libraries(${COMPONENT_TARGET} "-L ${CMAKE_CURRENT_SOURCE_DIR}/lib/esp32") + target_link_libraries(${COMPONENT_TARGET} "-L ${CMAKE_CURRENT_SOURCE_DIR}/esp-tts/esp_tts_chinese") + add_prebuilt_library(esp_audio_processor "${CMAKE_CURRENT_SOURCE_DIR}/lib/esp32/libesp_audio_processor.a" PRIV_REQUIRES esp-sr) add_prebuilt_library(wakenet "${CMAKE_CURRENT_SOURCE_DIR}/lib/esp32/libwakenet.a" PRIV_REQUIRES esp-sr) add_prebuilt_library(multinet "${CMAKE_CURRENT_SOURCE_DIR}/lib/esp32/libmultinet.a" PRIV_REQUIRES esp-sr) add_prebuilt_library(esp_audio_front_end "${CMAKE_CURRENT_SOURCE_DIR}/lib/esp32/libesp_audio_front_end.a" PRIV_REQUIRES esp-sr) - add_prebuilt_library(hufzip "${CMAKE_CURRENT_SOURCE_DIR}/lib/esp32/libhufzip.a" PRIV_REQUIRES esp-sr) - # TTS target_link_libraries(${COMPONENT_TARGET} "-Wl,--start-group" - esp_tts_chinese_esp32s2 - voice_set_xiaole_esp32s2 - voice_set_template_esp32s2 - "-Wl,--end-group") + wakenet + dl_lib + c_speech_features + hilexin_wn5 + hilexin_wn5X2 + hilexin_wn5X3 + nihaoxiaozhi_wn5 + nihaoxiaozhi_wn5X2 + nihaoxiaozhi_wn5X3 + nihaoxiaoxin_wn5X3 + customized_word_wn5 + multinet + multinet2_ch + esp_audio_processor + esp_audio_front_end + esp_tts_chinese + voice_set_xiaole + voice_set_template + "-Wl,--end-group") elseif(${IDF_TARGET} STREQUAL "esp32s2") target_link_libraries(${COMPONENT_TARGET} "-Wl,--start-group" esp_tts_chinese_esp32s2 @@ -34,6 +49,7 @@ elseif(${IDF_TARGET} STREQUAL "esp32s2") "-Wl,--end-group") elseif(${IDF_TARGET} STREQUAL "esp32s3") target_link_libraries(${COMPONENT_TARGET} "-L ${CMAKE_CURRENT_SOURCE_DIR}/lib/esp32s3") + add_prebuilt_library(esp_audio_processor "${CMAKE_CURRENT_SOURCE_DIR}/lib/esp32s3/libesp_audio_processor.a" PRIV_REQUIRES esp-sr) add_prebuilt_library(wakenet "${CMAKE_CURRENT_SOURCE_DIR}/lib/esp32s3/libwakenet.a" PRIV_REQUIRES esp-sr) add_prebuilt_library(multinet "${CMAKE_CURRENT_SOURCE_DIR}/lib/esp32s3/libmultinet.a" PRIV_REQUIRES esp-sr) diff --git a/Kconfig.projbuild b/Kconfig.projbuild index 72c111d..5280a58 100644 --- a/Kconfig.projbuild +++ b/Kconfig.projbuild @@ -27,6 +27,7 @@ endchoice choice MODEL_DATA_PATH prompt "model data path" default MODEL_IN_SPIFFS + depends on IDF_TARGET_ESP32S3 config MODEL_IN_SPIFFS bool "spiffs partition" @@ -101,11 +102,7 @@ choice SR_WN_WAKE_WORD_SEL config SR_WN_WN5X3_NIHAOXIAOZHI bool "nihaoxiaozhi (WakeNet5X3)" - depends on SR_WN_MODEL_WN5_QUANT || SR_WN_MODEL_WN5_FLOAT - - config SR_WN_WN5X3_HIJESON - bool "hi jeson (WakeNet5X3)" - depends on SR_WN_MODEL_WN5_QUANT || SR_WN_MODEL_WN5_FLOAT + depends on SR_WN_MODEL_WN5_QUANT || SR_WN_MODEL_WN5_FLOAT config SR_WN_WN5X3_NIHAOXIAOXIN bool "nihaoxiaoxin (WakeNet5X3)" @@ -157,6 +154,7 @@ choice SR_MN_LANGUAGE_SEL config SR_MN_ENGLISH bool "english" + depends on IDF_TARGET_ESP32S3 endchoice @@ -187,10 +185,6 @@ choice SR_MN_MODE_SEL bool "chinese single recognition (MultiNet2)" depends on SR_MN_CHINESE && IDF_TARGET_ESP32 - config SR_MN_CN_MULTINET2_CONTINUOUS_RECOGNITION - bool "chinese continuous recognition (MultiNet2)" - depends on SR_MN_CHINESE && IDF_TARGET_ESP32 - config SR_MN_CN_MULTINET3_SINGLE_RECOGNITION bool "chinese single recognition (MultiNet3)" depends on SR_MN_CHINESE && IDF_TARGET_ESP32S3 diff --git a/include/customized_word_wn5.h b/include/customized_word_wn5.h new file mode 100644 index 0000000..57199e5 --- /dev/null +++ b/include/customized_word_wn5.h @@ -0,0 +1,8 @@ +//Generated by mkmodel +#pragma once +#include +#include "dl_lib_coefgetter_if.h" +#include "dl_lib_matrix.h" +#include "dl_lib_matrixq.h" + +extern const model_coeff_getter_t get_coeff_customized_word_wn5; diff --git a/include/dl_lib.h b/include/dl_lib.h index bba0e2c..5afe563 100644 --- a/include/dl_lib.h +++ b/include/dl_lib.h @@ -20,12 +20,11 @@ typedef int padding_state; -#ifdef CONFIG_IDF_TARGET_ESP32S3 void *dl_lib_calloc(int cnt, int size, int align); + void *dl_lib_calloc_psram(int cnt, int size, int align); void dl_lib_free(void *d); -#endif /** * @brief Does a fast version of the exp() operation on a floating point number. * diff --git a/include/esp_afe_sr_iface.h b/include/esp_afe_sr_iface.h index f27ea55..567d967 100644 --- a/include/esp_afe_sr_iface.h +++ b/include/esp_afe_sr_iface.h @@ -41,6 +41,25 @@ typedef struct { int agc_mode; } afe_config_t; + +#if CONFIG_IDF_TARGET_ESP32 +#define AFE_CONFIG_DEFAULT() { \ + .aec_init = true, \ + .se_init = true, \ + .vad_init = true, \ + .wakenet_init = true, \ + .vad_mode = 3, \ + .wakenet_model = &WAKENET_MODEL, \ + .wakenet_coeff = &WAKENET_COEFF, \ + .wakenet_mode = DET_MODE_90, \ + .afe_mode = SR_MODE_HIGH_PERF, \ + .afe_perferred_core = 0, \ + .afe_perferred_priority = 5, \ + .afe_ringbuf_size = 50, \ + .alloc_from_psram = 1, \ + .agc_mode = 2, \ +} +#elif CONFIG_IDF_TARGET_ESP32S3 #define AFE_CONFIG_DEFAULT() { \ .aec_init = true, \ .se_init = true, \ @@ -57,7 +76,7 @@ typedef struct { .alloc_from_psram = 1, \ .agc_mode = 2, \ } - +#endif /** * @brief Function to initialze a AFE_SR instance with a specified mode * diff --git a/include/hijeson_wn5X3.h b/include/hijeson_wn5X3.h new file mode 100644 index 0000000..c2e3438 --- /dev/null +++ b/include/hijeson_wn5X3.h @@ -0,0 +1,8 @@ +//Generated by mkmodel +#pragma once +#include +#include "dl_lib_coefgetter_if.h" +#include "dl_lib_matrix.h" +#include "dl_lib_matrixq.h" + +extern const model_coeff_getter_t get_coeff_hijeson_wn5X3; diff --git a/include/hilexin_wn5.h b/include/hilexin_wn5.h new file mode 100644 index 0000000..d922a6a --- /dev/null +++ b/include/hilexin_wn5.h @@ -0,0 +1,8 @@ +//Generated by mkmodel +#pragma once +#include +#include "dl_lib_coefgetter_if.h" +#include "dl_lib_matrix.h" +#include "dl_lib_matrixq.h" + +extern const model_coeff_getter_t get_coeff_hilexin_wn5; diff --git a/include/hilexin_wn5X2.h b/include/hilexin_wn5X2.h new file mode 100644 index 0000000..5ca6bbc --- /dev/null +++ b/include/hilexin_wn5X2.h @@ -0,0 +1,8 @@ +//Generated by mkmodel +#pragma once +#include +#include "dl_lib_coefgetter_if.h" +#include "dl_lib_matrix.h" +#include "dl_lib_matrixq.h" + +extern const model_coeff_getter_t get_coeff_hilexin_wn5X2; diff --git a/include/hilexin_wn5X3.h b/include/hilexin_wn5X3.h new file mode 100644 index 0000000..c78a64d --- /dev/null +++ b/include/hilexin_wn5X3.h @@ -0,0 +1,8 @@ +//Generated by mkmodel +#pragma once +#include +#include "dl_lib_coefgetter_if.h" +#include "dl_lib_matrix.h" +#include "dl_lib_matrixq.h" + +extern const model_coeff_getter_t get_coeff_hilexin_wn5X3; diff --git a/include/multinet2_ch.h b/include/multinet2_ch.h new file mode 100644 index 0000000..2cee215 --- /dev/null +++ b/include/multinet2_ch.h @@ -0,0 +1,9 @@ +//Generated by mkmodel_py +#pragma once +#include +#include "dl_lib_coefgetter_if.h" +#include "dl_lib_matrix.h" +#include "dl_lib_matrixq.h" +#include "dl_lib_matrixq8.h" + +extern const model_coeff_getter_t get_coeff_multinet2_ch; \ No newline at end of file diff --git a/include/nihaoxiaoxin_wn5X3.h b/include/nihaoxiaoxin_wn5X3.h new file mode 100644 index 0000000..b767bfc --- /dev/null +++ b/include/nihaoxiaoxin_wn5X3.h @@ -0,0 +1,8 @@ +//Generated by mkmodel +#pragma once +#include +#include "dl_lib_coefgetter_if.h" +#include "dl_lib_matrix.h" +#include "dl_lib_matrixq.h" + +extern const model_coeff_getter_t get_coeff_nihaoxiaoxin_wn5X3; diff --git a/include/nihaoxiaoxin_wn6.h b/include/nihaoxiaoxin_wn6.h new file mode 100644 index 0000000..c365e6d --- /dev/null +++ b/include/nihaoxiaoxin_wn6.h @@ -0,0 +1,8 @@ +//Generated by mkmodel +#pragma once +#include +#include "dl_lib_coefgetter_if.h" +#include "dl_lib_matrix.h" +#include "dl_lib_matrixq.h" + +extern const model_coeff_getter_t get_coeff_nihaoxiaoxin_wn6; diff --git a/include/nihaoxiaozhi_wn5.h b/include/nihaoxiaozhi_wn5.h new file mode 100644 index 0000000..04fa268 --- /dev/null +++ b/include/nihaoxiaozhi_wn5.h @@ -0,0 +1,8 @@ +//Generated by mkmodel +#pragma once +#include +#include "dl_lib_coefgetter_if.h" +#include "dl_lib_matrix.h" +#include "dl_lib_matrixq.h" + +extern const model_coeff_getter_t get_coeff_nihaoxiaozhi_wn5; diff --git a/include/nihaoxiaozhi_wn5X2.h b/include/nihaoxiaozhi_wn5X2.h new file mode 100644 index 0000000..6c5f2b3 --- /dev/null +++ b/include/nihaoxiaozhi_wn5X2.h @@ -0,0 +1,8 @@ +//Generated by mkmodel +#pragma once +#include +#include "dl_lib_coefgetter_if.h" +#include "dl_lib_matrix.h" +#include "dl_lib_matrixq.h" + +extern const model_coeff_getter_t get_coeff_nihaoxiaozhi_wn5X2; diff --git a/include/nihaoxiaozhi_wn5X3.h b/include/nihaoxiaozhi_wn5X3.h new file mode 100644 index 0000000..190556e --- /dev/null +++ b/include/nihaoxiaozhi_wn5X3.h @@ -0,0 +1,8 @@ +//Generated by mkmodel +#pragma once +#include +#include "dl_lib_coefgetter_if.h" +#include "dl_lib_matrix.h" +#include "dl_lib_matrixq.h" + +extern const model_coeff_getter_t get_coeff_nihaoxiaozhi_wn5X3; diff --git a/lib/esp32/libc_speech_features.a b/lib/esp32/libc_speech_features.a new file mode 100644 index 0000000..ced10ea Binary files /dev/null and b/lib/esp32/libc_speech_features.a differ diff --git a/lib/esp32/libcustomized_word_wn5.a b/lib/esp32/libcustomized_word_wn5.a new file mode 100644 index 0000000..6fc29aa Binary files /dev/null and b/lib/esp32/libcustomized_word_wn5.a differ diff --git a/lib/esp32/libdl_lib.a b/lib/esp32/libdl_lib.a new file mode 100644 index 0000000..b66cf42 Binary files /dev/null and b/lib/esp32/libdl_lib.a differ diff --git a/lib/esp32/libdl_lib_esp32.a b/lib/esp32/libdl_lib_esp32.a deleted file mode 100644 index c89d551..0000000 Binary files a/lib/esp32/libdl_lib_esp32.a and /dev/null differ diff --git a/lib/esp32/libesp_audio_front_end.a b/lib/esp32/libesp_audio_front_end.a new file mode 100644 index 0000000..af525a0 Binary files /dev/null and b/lib/esp32/libesp_audio_front_end.a differ diff --git a/lib/esp32/libesp_audio_processor.a b/lib/esp32/libesp_audio_processor.a new file mode 100644 index 0000000..f168bc6 Binary files /dev/null and b/lib/esp32/libesp_audio_processor.a differ diff --git a/lib/esp32/libhilexin_wn5.a b/lib/esp32/libhilexin_wn5.a new file mode 100644 index 0000000..fc3decf Binary files /dev/null and b/lib/esp32/libhilexin_wn5.a differ diff --git a/lib/esp32/libhilexin_wn5X2.a b/lib/esp32/libhilexin_wn5X2.a new file mode 100644 index 0000000..5b5200b Binary files /dev/null and b/lib/esp32/libhilexin_wn5X2.a differ diff --git a/lib/esp32/libhilexin_wn5X3.a b/lib/esp32/libhilexin_wn5X3.a new file mode 100644 index 0000000..812d083 Binary files /dev/null and b/lib/esp32/libhilexin_wn5X3.a differ diff --git a/lib/esp32/libmultinet.a b/lib/esp32/libmultinet.a new file mode 100644 index 0000000..1ff6931 Binary files /dev/null and b/lib/esp32/libmultinet.a differ diff --git a/lib/esp32/libmultinet2_ch.a b/lib/esp32/libmultinet2_ch.a new file mode 100644 index 0000000..076be7a Binary files /dev/null and b/lib/esp32/libmultinet2_ch.a differ diff --git a/lib/esp32/libnihaoxiaoxin_wn5X3.a b/lib/esp32/libnihaoxiaoxin_wn5X3.a new file mode 100644 index 0000000..6107a76 Binary files /dev/null and b/lib/esp32/libnihaoxiaoxin_wn5X3.a differ diff --git a/lib/esp32/libnihaoxiaozhi_wn5.a b/lib/esp32/libnihaoxiaozhi_wn5.a new file mode 100644 index 0000000..609097d Binary files /dev/null and b/lib/esp32/libnihaoxiaozhi_wn5.a differ diff --git a/lib/esp32/libnihaoxiaozhi_wn5X2.a b/lib/esp32/libnihaoxiaozhi_wn5X2.a new file mode 100644 index 0000000..f4039dd Binary files /dev/null and b/lib/esp32/libnihaoxiaozhi_wn5X2.a differ diff --git a/lib/esp32/libnihaoxiaozhi_wn5X3.a b/lib/esp32/libnihaoxiaozhi_wn5X3.a new file mode 100644 index 0000000..0b166f8 Binary files /dev/null and b/lib/esp32/libnihaoxiaozhi_wn5X3.a differ diff --git a/lib/esp32/libwakenet.a b/lib/esp32/libwakenet.a new file mode 100644 index 0000000..b12c6c6 Binary files /dev/null and b/lib/esp32/libwakenet.a differ diff --git a/libversion b/libversion index 8e23329..2de1b26 100644 --- a/libversion +++ b/libversion @@ -1 +1 @@ -d187cb93b72cefcef1904550d50c5f1868bb0099 \ No newline at end of file +695cd3da8e08da040b6f69bce2e55c3f3c77a902 \ No newline at end of file diff --git a/src/esp_afe_sr_1mic.c b/src/esp_afe_sr_1mic.c index 7963db0..9f7c221 100644 --- a/src/esp_afe_sr_1mic.c +++ b/src/esp_afe_sr_1mic.c @@ -95,14 +95,20 @@ static esp_afe_sr_data_t *afe_create_from_config(afe_config_t *afe_config) } if (afe->mode == SR_MODE_LOW_COST) { +#ifdef CONFIG_IDF_TARGET_ESP32S3 afe->aec_handle = esp_aec3_728_init(afe->nch, afe->aec_frame_size, afe->aec_filter_length, afe_config->alloc_from_psram); +#else + printf("Not support SR_MODE_LOW_COST on ESP32\n"); + afe->aec_handle = NULL; +#endif } else { afe->aec_handle = esp_aec3_init(afe->nch, afe->aec_frame_size, - afe->aec_filter_length, 1); + afe->aec_filter_length, 0); } } else { afe->aec_frame_size = 512; + afe->aec_nch = afe->nch + 1; afe->aec_handle = NULL; } @@ -159,7 +165,11 @@ static esp_afe_sr_data_t *afe_create_from_config(afe_config_t *afe_config) afe->wn_gain = 2; afe->agc_mode = 0; } - afe->buff_wn = malloc(afe->audio_chunksize * (afe->wn_nch + afe->wn_nch - afe->nch) * sizeof(int16_t)); + if (afe_config->alloc_from_psram) { + afe->buff_wn = heap_caps_malloc(afe->audio_chunksize * (afe->wn_nch + afe->wn_nch - afe->nch) * sizeof(int16_t), MALLOC_CAP_SPIRAM); + } else { + afe->buff_wn = malloc(afe->audio_chunksize * (afe->wn_nch + afe->wn_nch - afe->nch) * sizeof(int16_t)); + } /********************************************************/ /**********************INIT OTRHERS**********************/ @@ -209,8 +219,10 @@ static int afe_feed(esp_afe_sr_data_t *afe, int16_t *in) if (frame_cnt++ < 4) ref_vad_res = 0; if (afe->mode == SR_MODE_LOW_COST) { +#ifdef CONFIG_IDF_TARGET_ESP32S3 esp_aec3_728_write_ref_vad(afe->aec_handle, ref_vad_res); esp_aec3_728_process(afe->aec_handle, afe->aec_in, afe->aec_in + afe->aec_frame_size, afe->aec_tmp); +#endif } else if (afe->mode == SR_MODE_HIGH_PERF) { esp_aec3_write_ref_vad(afe->aec_handle, ref_vad_res); esp_aec3_process(afe->aec_handle, afe->aec_in, afe->aec_in + afe->aec_frame_size, afe->aec_tmp); @@ -266,7 +278,9 @@ static void afe_se_task(void *arg) } } } - + // if (sr_rb_available(afe->rb_out) < afe->wn_nch * afe->ns_frame_size * sizeof(int16_t)) { + // printf("ERROR! rb_out slow!!!\n"); + // } sr_rb_write(afe->rb_out, afe->ns_in, afe->wn_nch * afe->ns_frame_size * sizeof(int16_t), 0); } printf("afe_se_task quit\n"); @@ -302,9 +316,9 @@ static int afe_fetch(esp_afe_sr_data_t *afe, int16_t *out) res = afe->wakenet->detect(afe->model_data, afe->buff_wn); //selector & gainer afe->channel_id = afe->wakenet->get_triggered_channel(afe->model_data); - if (res > 0 && afe->agc_mode>=0) { + if (res > 0 && afe->agc_mode >= 0) { if (afe->agc_mode == 0) out_gain = 1; - else out_gain = afe->wakenet->get_vol_gain(afe->model_data, (afe->agc_mode-6)); + else out_gain = afe->wakenet->get_vol_gain(afe->model_data, (afe->agc_mode - 6)); } int shift = afe->audio_chunksize * afe->channel_id; @@ -435,8 +449,10 @@ static void afe_destory(esp_afe_sr_data_t *afe) if (afe->aec_handle != NULL) { if (afe->mode == SR_MODE_LOW_COST) { +#ifdef CONFIG_IDF_TARGET_ESP32S3 vad_destroy(afe->ref_vad_handle); esp_aec3_728_destroy(afe->aec_handle); +#endif } else if (afe->mode == SR_MODE_HIGH_PERF) { vad_destroy(afe->ref_vad_handle); @@ -454,12 +470,12 @@ static void afe_destory(esp_afe_sr_data_t *afe) if (afe->wakenet != NULL) { afe->wakenet->destroy(afe->model_data); - if (afe->buff_wn != NULL) { - free(afe->buff_wn); - } - afe->wakenet = NULL; } + if (afe->buff_wn != NULL) { + free(afe->buff_wn); + afe->buff_wn = NULL; + } if (afe->vad_handle != NULL) { vad_destroy(afe->vad_handle);