mirror of
https://github.com/modelscope/FunASR
synced 2025-09-15 14:48:36 +08:00
f47d43c020
* resume from step * batch * batch * batch * batch * batch * batch * batch * batch * batch * batch * batch * batch * batch * batch * batch * train_loss_avg train_acc_avg * train_loss_avg train_acc_avg * train_loss_avg train_acc_avg * log step * wav is not exist * wav is not exist * decoding * decoding * decoding * wechat * decoding key * decoding key * decoding key * decoding key * decoding key * decoding key * dynamic batch * start_data_split_i=0 * total_time/accum_grad * total_time/accum_grad * total_time/accum_grad * update avg slice * update avg slice * sensevoice sanm * sensevoice sanm * add * add * add * add * deepspeed * update with main (#1731) * c++ runtime adapt to 1.0 (#1724) * adapt vad runtime to 1.0 * add json * change yml name * add func LoadVocabFromJson * add token file for InitAsr * add token path for OfflineStream * add funcOpenYaml * add token file for InitPunc * add token file for stream * update punc-model * update funasr-wss-server * update runtime_sdk_download_tool.py * update docker list * Delete docs/images/wechat.png * Add files via upload * Emo2Vec限定选择的情感类别 (#1730) * 限定选择的情感类别 * 使用none来禁用情感标签输出 * 修改输出接口 * 使用unuse来禁用token --------- Co-authored-by: 常材 <gaochangfeng.gcf@alibaba-inc.com> * bugfix * v1.0.27 * update docs * hf hub * Fix incorrect assignment of 'end' attribute to 'start' in sentences list comprehension (#1680) --------- Co-authored-by: Yabin Li <wucong.lyb@alibaba-inc.com> Co-authored-by: gaochangfeng <54253717+gaochangfeng@users.noreply.github.com> Co-authored-by: 常材 <gaochangfeng.gcf@alibaba-inc.com> Co-authored-by: nsdou <168500039+nsdou@users.noreply.github.com> * docs * docs * deepspeed * deepspeed * deepspeed * deepspeed * update * ds * ds * ds * ds * ds * ds * ds * add * add * bugfix * add * wenetspeech * wenetspeech * wenetspeech * wenetspeech * wenetspeech * wenetspeech --------- Co-authored-by: Yabin Li <wucong.lyb@alibaba-inc.com> Co-authored-by: gaochangfeng <54253717+gaochangfeng@users.noreply.github.com> Co-authored-by: 常材 <gaochangfeng.gcf@alibaba-inc.com> Co-authored-by: nsdou <168500039+nsdou@users.noreply.github.com>
67 lines
2.0 KiB
Bash
Executable File
67 lines
2.0 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Copyright 2017 Xingyu Na
|
|
# Apache 2.0
|
|
|
|
#. ./path.sh || exit 1;
|
|
|
|
if [ $# != 3 ]; then
|
|
echo "Usage: $0 <audio-path> <text-path> <output-path>"
|
|
echo " $0 /export/a05/xna/data/data_aishell/wav /export/a05/xna/data/data_aishell/transcript data"
|
|
exit 1;
|
|
fi
|
|
|
|
aishell_audio_dir=$1
|
|
aishell_text=$2/aishell_transcript_v0.8.txt
|
|
output_dir=$3
|
|
|
|
train_dir=$output_dir/data/local/train
|
|
dev_dir=$output_dir/data/local/dev
|
|
test_dir=$output_dir/data/local/test
|
|
tmp_dir=$output_dir/data/local/tmp
|
|
|
|
mkdir -p $train_dir
|
|
mkdir -p $dev_dir
|
|
mkdir -p $test_dir
|
|
mkdir -p $tmp_dir
|
|
|
|
# data directory check
|
|
if [ ! -d $aishell_audio_dir ] || [ ! -f $aishell_text ]; then
|
|
echo "Error: $0 requires two directory arguments"
|
|
exit 1;
|
|
fi
|
|
|
|
# find wav audio file for train, dev and test resp.
|
|
find $aishell_audio_dir -iname "*.wav" > $tmp_dir/wav.flist
|
|
n=`cat $tmp_dir/wav.flist | wc -l`
|
|
[ $n -ne 141925 ] && \
|
|
echo Warning: expected 141925 data data files, found $n
|
|
|
|
grep -i "wav/train" $tmp_dir/wav.flist > $train_dir/wav.flist || exit 1;
|
|
grep -i "wav/dev" $tmp_dir/wav.flist > $dev_dir/wav.flist || exit 1;
|
|
grep -i "wav/test" $tmp_dir/wav.flist > $test_dir/wav.flist || exit 1;
|
|
|
|
rm -r $tmp_dir
|
|
|
|
# Transcriptions preparation
|
|
for dir in $train_dir $dev_dir $test_dir; do
|
|
echo Preparing $dir transcriptions
|
|
sed -e 's/\.wav//' $dir/wav.flist | awk -F '/' '{print $NF}' > $dir/utt.list
|
|
paste -d' ' $dir/utt.list $dir/wav.flist > $dir/wav.scp_all
|
|
utils/filter_scp.pl -f 1 $dir/utt.list $aishell_text > $dir/transcripts.txt
|
|
awk '{print $1}' $dir/transcripts.txt > $dir/utt.list
|
|
utils/filter_scp.pl -f 1 $dir/utt.list $dir/wav.scp_all | sort -u > $dir/wav.scp
|
|
sort -u $dir/transcripts.txt > $dir/text
|
|
done
|
|
|
|
mkdir -p $output_dir/data/train $output_dir/data/dev $output_dir/data/test
|
|
|
|
for f in wav.scp text; do
|
|
cp $train_dir/$f $output_dir/data/train/$f || exit 1;
|
|
cp $dev_dir/$f $output_dir/data/dev/$f || exit 1;
|
|
cp $test_dir/$f $output_dir/data/test/$f || exit 1;
|
|
done
|
|
|
|
echo "$0: AISHELL data preparation succeeded"
|
|
exit 0;
|