From dcd40590df7ab625b7934694c13d3d5fbc941f79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= <zhifu.gzf@alibaba-inc.com>
Date: Sun, 9 Jun 2024 01:08:29 +0800
Subject: [PATCH] fix bug

---
 .../llm_asr/conf/whisper_qwen_linear2.yaml    | 25 +++++++++++--------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/examples/industrial_data_pretraining/llm_asr/conf/whisper_qwen_linear2.yaml b/examples/industrial_data_pretraining/llm_asr/conf/whisper_qwen_linear2.yaml
index 62d77e260..c2f641977 100644
--- a/examples/industrial_data_pretraining/llm_asr/conf/whisper_qwen_linear2.yaml
+++ b/examples/industrial_data_pretraining/llm_asr/conf/whisper_qwen_linear2.yaml
@@ -59,17 +59,20 @@ scheduler_conf:
 
 dataset: OpenAIDataset
 dataset_conf:
-    index_ds: OpenAIIndexDSJsonl
-    batch_sampler: CustomDistributedBatchSampler
-    batch_type: example # example or length
-    batch_size: 4 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
-    max_token_length: 3000 # filter samples if source_token_len+target_token_len > max_token_length,
-    shuffle: True
-    num_workers: 0
-    audio_adaptor_downsample_rate: ${audio_adaptor_conf.downsample_rate}
-    audio_encoder_downsample_rate: 2
-#    prompt: "<|startoftranscription|><|zh|><|transcribe|><|zh|><|notimestamps|><|wo_itn|>"
-
+  index_ds: OpenAIIndexDSJsonl
+  batch_sampler: BatchSampler
+  batch_type: token
+  batch_size: 900
+  max_token_length: 1024
+  shuffle: true
+  sort_size: 1024
+  batch_size_scale_ratio_max: 2
+  num_workers: 4
+  audio_adaptor_downsample_rate: ${audio_adaptor_conf.downsample_rate}
+  audio_encoder_downsample_rate: 2
+  data_split_num: 512
+  batch_size_sample_max: 15
+  retry: 20
 
 
 tokenizer: HuggingfaceTokenizer